Browse Source

ruby: add auxiliar scripts in order to help tests

ruby_find_pkgsdeps: look for file dependencies (checks
require and Encoding references) and extrapolate it to pkgs
deps. Also checks whether a dep is redundant or missing in pkgs.
Must run inside an OpenWRT with all ruby* pkgs installed.

ruby_missingfiles: list files in staging/target and from files
comparing side by side its contents. It helps to easly visualize
which file is not packaged in an ipk.

Signed-off-by: Luiz Angelo Daros de Luca <luizluca@gmail.com>
Luiz Angelo Daros de Luca 10 years ago
parent
commit
e707020e18
2 changed files with 285 additions and 0 deletions
  1. 265
    0
      lang/ruby/ruby_find_pkgsdeps
  2. 20
    0
      lang/ruby/ruby_missingfiles

+ 265
- 0
lang/ruby/ruby_find_pkgsdeps View File

1
+#!/usr/bin/ruby -Eutf-8
2
+# encoding: utf-8
3
+#
4
+# Find dependencies between ruby packages
5
+#
6
+# Must run inside a openwrt with all *ruby* packages installed
7
+#
8
+
9
+failed = false
10
+
11
+puts "Looking for installed ruby packages..."
12
+packages=`opkg list-installed '*ruby*' | cut -d' ' -f 1`.split("\n")
13
+
14
+puts "Looking for packages files..."
15
+package_files=Hash.new([])
16
+packages.each do
17
+	|pkg|
18
+	files=`opkg files "#{pkg}" | sed -e 1d`.split("\n")
19
+	package_files[pkg]=files if files
20
+end
21
+
22
+require_regex=/^require ["']([^"']+)["'].*/
23
+require_regex_ignore=/^require ([a-zA-Z\$]|["']$|.*\/$)/
24
+require_ignore=%w{drb/invokemethod16 foo rubygems/defaults/operating_system win32console java Win32API
25
+                  builder/xchar json/pure simplecov win32/sspi rdoc/markdown/literals_1_8 enumerator win32/resolv rbtree
26
+                  nqxml/streamingparser nqxml/treeparser xmlscan/parser xmlscan/scanner xmltreebuilder xml/parser xmlparser xml/encoding-ja xmlencoding-ja
27
+                  iconv uconv}
28
+
29
+builtin_enc=[
30
+	Encoding.find("ASCII-8BIT"),
31
+	Encoding.find("UTF-8"),
32
+	Encoding.find("UTF-7"),
33
+	Encoding.find("US-ASCII"),
34
+]
35
+
36
+puts "Looking for requires in files..."
37
+files_requires=Hash.new([])
38
+packages.each do
39
+        |pkg|
40
+	package_files[pkg].each do
41
+		|file|
42
+		next if not File.file?(file)
43
+
44
+		if not file =~ /.rb$/
45
+			if File.executable?(file)
46
+				magic=`head -c50 '#{file}' | head -1`
47
+				begin
48
+					if not magic =~ /ruby/
49
+						next
50
+					end
51
+				rescue
52
+					next
53
+				end
54
+			else
55
+				next
56
+			end
57
+		end
58
+		#puts "Checking #{file}..."
59
+		File.open(file, "r") do
60
+			|f|
61
+			lineno=0
62
+			while line=f.gets() do
63
+				lineno+=1; encs=[]; requires=[]; need_encdb=false
64
+
65
+				line=line.chomp.gsub!(/^[[:blank:]]*/,"")
66
+
67
+				case line
68
+				when /^#.*coding *:/
69
+					if lineno <= 2
70
+						enc=line.sub(/.*coding *: */,"").sub(/ .*/,"")
71
+						encs << Encoding.find(enc)
72
+					end
73
+				end
74
+				line.gsub!(/#.*/,"")
75
+				case line
76
+				when "__END__"
77
+					break
78
+				when /^require /
79
+					#puts "#{file}:#{line}"
80
+					if require_regex_ignore =~ line
81
+						#puts "Ignoring #{line} at #{file}:#{lineno} (REGEX)..."
82
+						next
83
+					end
84
+					if not require_regex =~ line
85
+						$stderr.puts "Unknown require: '#{line}' at file #{file}:#{lineno}"
86
+						failed=true
87
+					end
88
+					require=line.gsub(require_regex,"\\1")
89
+					require.gsub!(/\.(so|rb)$/,"")
90
+
91
+					if require_ignore.include?(require)
92
+						#puts "Ignoring #{line} at #{file}:#{lineno} (STR)..."
93
+						next
94
+					end
95
+
96
+					files_requires[file]=files_requires[file] + [require]
97
+
98
+				when /Encoding::/
99
+					encs=line.scan(/Encoding::[[:alnum:]_]+/).collect {|enc| eval(enc) }.select {|enc| enc.kind_of? Encoding }
100
+					need_encdb=true
101
+				end
102
+
103
+				next if encs.empty?
104
+				required_encs = (encs - builtin_enc).collect {|enc| "enc/#{enc.name.downcase.gsub("-","_")}" }
105
+				required_encs << "enc/encdb" if need_encdb
106
+
107
+				files_requires[file] = files_requires[file] + required_encs
108
+			end
109
+		end
110
+	end
111
+end
112
+exit(1) if failed
113
+
114
+# Add deps from .so
115
+package_files.each do |(pkg,files)| files.each do |file|
116
+	case file
117
+	when /\/nkf\.so$/
118
+		files_requires[file]= files_requires[file] + ["enc/encdb"]
119
+	end
120
+end; end
121
+
122
+puts "Merging requirements into packages..."
123
+package_requires = Hash[packages.collect { |pkg| [pkg, package_files[pkg].collect {|file| files_requires[file] }.inject([],:+).uniq] }]
124
+
125
+weak_dependency=Hash.new([])
126
+weak_dependency.merge!({
127
+"ruby-misc"=>["ruby-openssl"], 					#securerandom.rb
128
+"ruby-debuglib"=>["ruby-readline"],				#debug.rb
129
+"ruby-drb"=>["ruby-openssl"],				 	#drb/ssl.rb
130
+"ruby-irb"=>["ruby-rdoc", "ruby-readline"],		 	#irb/cmd/help.rb
131
+"ruby-gems"=>["ruby-openssl","ruby-io-console", 		#rubygems/commands/cert_command.rb rubygems/user_interaction.rb
132
+	      "ruby-minitest", "ruby-webrick"],			#rubygems/test_case.rb rubygems/server.rb
133
+"ruby-mkmf"=>["ruby-webrick"], 					#un.rb
134
+"ruby-net"=>["ruby-openssl","ruby-io-console","ruby-zlib"], 	#net/*.rb
135
+"ruby-optparse"=>["ruby-uri","ruby-datetime"],			#optparse/date.rb optparse/uri.rb
136
+"ruby-rake"=>["ruby-net","ruby-testunit","ruby-gems"],		#rake/contrib/ftptools.rb rake/runtest.rb /usr/bin/rake
137
+"ruby-rdoc"=>["ruby-gems","ruby-readline","ruby-webrick",
138
+	      "ruby-minitest"],					#/usr/bin/rdoc and others
139
+"ruby-testunit"=>["ruby-gems", "ruby-io-console"],		#test/unit/parallel.rb test/unit.rb
140
+"ruby-webrick"=>["ruby-openssl"],				#webrick/ssl.rb
141
+})
142
+
143
+puts "Looking for package dependencies..."
144
+package_provides = {}
145
+package_dependencies = Hash.new([])
146
+package_requires.each do
147
+	|(pkg,requires)|
148
+
149
+	requires.each do
150
+		|require|
151
+		if package_provides.include?(require)
152
+			found = package_provides[require]
153
+		else
154
+			found = package_files.detect {|(pkg,files)| files.detect {|file| $:.detect {|path| "#{path}/#{require}" == file.gsub(/\.(so|rb)$/,"") } } }
155
+			if not found
156
+				$stderr.puts "#{pkg}: Nobody provides #{require}"
157
+				failed = true
158
+				next
159
+			end
160
+			found = found.first
161
+			package_provides[require]=found
162
+		end
163
+		if weak_dependency[pkg].include?(found)
164
+			puts "#{pkg}: #{found} provides #{require} (ignored WEAK dep)"
165
+		else
166
+			puts "#{pkg}: #{found} provides #{require}"
167
+			package_dependencies[pkg]=package_dependencies[pkg] + [found]
168
+		end
169
+	end
170
+end
171
+exit(1) if failed
172
+
173
+package_dependencies.each do
174
+        |(pkg,deps)|
175
+        package_dependencies[pkg]=deps.uniq.sort - [pkg]
176
+end
177
+
178
+puts "Expanding dependencies..."
179
+begin
180
+	changed=false
181
+	package_dependencies.each do
182
+		|(pkg,deps)|
183
+
184
+		next if deps.empty?
185
+
186
+		deps_new = deps.collect {|dep| [dep] + package_dependencies[dep] }.inject([],:+).uniq.sort
187
+		if not deps == deps_new
188
+			puts "#{pkg}: #{deps.join(",")}"
189
+			puts "#{pkg}: #{deps_new.join(",")}"
190
+			package_dependencies[pkg]=deps_new
191
+			changed=true
192
+		end
193
+	end
194
+end if not changed
195
+
196
+puts "Checking for mutual dependencies..."
197
+package_dependencies.each do
198
+	|(pkg,deps)|
199
+	if deps.include? pkg
200
+		$stderr.puts "#{pkg}: Cycle dependency detected! "
201
+		failed = true
202
+	end
203
+end
204
+exit(1) if failed
205
+
206
+puts "Removing redundant dependencies..."
207
+package_dependencies.each do
208
+	|(pkg,deps)|
209
+	package_dependencies[pkg]=deps.uniq - [pkg]
210
+end
211
+
212
+package_dependencies2=package_dependencies.dup
213
+package_dependencies.each do
214
+	|(pkg,deps)|
215
+
216
+	# Ignore dependencies that are aready required by another dependency
217
+	deps_clean = deps.reject {|dep_suspect| deps.detect {|dep_provider|
218
+			if package_dependencies[dep_provider].include?(dep_suspect)
219
+				puts "#{pkg}: #{dep_suspect} is already required by #{dep_provider}"
220
+				true
221
+			end
222
+		 } }
223
+
224
+	if not deps==deps_clean
225
+		puts "before: #{deps.join(",")}"
226
+		puts "after: #{deps_clean.join(",")}"
227
+		package_dependencies2[pkg]=deps_clean
228
+	end
229
+end
230
+package_dependencies=package_dependencies2
231
+
232
+puts "Checking current packages dependencies..."
233
+ok=true
234
+package_dependencies.each do
235
+	|(pkg,deps)|
236
+	current_deps=`opkg depends #{pkg} | sed -r -e '1d;s/^[[:blank:]]*//'`.split("\n")
237
+	current_deps.reject!{|dep| dep =~ /^lib/ }
238
+	current_deps -= ["ruby"]
239
+
240
+	extra_dep = current_deps - deps
241
+	$stderr.puts "Package #{pkg} does not need to depend on #{extra_dep.join(" ")} " if not extra_dep.empty?
242
+	missing_dep = deps - current_deps
243
+	$stderr.puts "Package #{pkg} need to depend on #{missing_dep.join(" ")} " if not missing_dep.empty?
244
+
245
+	if not extra_dep.empty? or not missing_dep.empty?
246
+		$stderr.puts "define Package/#{pkg}"
247
+		$stderr.puts "  DEPENDS:=ruby#{([""] +deps).join(" +")}"
248
+		ok=false
249
+	end
250
+end
251
+
252
+puts "All dependencies are OK." if ok
253
+
254
+
255
+__END__
256
+
257
+puts RUBY_VERSION, RUBY_PLATFORM
258
+puts 123
259
+
260
+puts Object.contants
261
+
262
+#RUBY_VER=2.1
263
+#RUBY_ARCH=i486-linux-gnu
264
+#RUBYLIB=/usr/lib/ruby/$RUBY_VER/
265
+#RUBYLIB_A=/usr/lib/ruby/$RUBY_ARCH/$RUBY_VER/

+ 20
- 0
lang/ruby/ruby_missingfiles View File

1
+#!/bin/bash
2
+#
3
+
4
+function list_staging_files {
5
+	cd staging_dir/target-*/; find \
6
+		\( \( -name "root-x86" -or -name "packages" -or -name "stamp" -or -name "pkginfo" \) -prune \) -or -true \
7
+		\( -path "*ruby*" -or -name "erb" -or -name "gem" -or -name "irb" -or -name "rake" -or -name "rdoc" -or -name "ri" -or -name "testrb" \) \
8
+		-print | sort
9
+}
10
+
11
+function list_ipkg_files {
12
+	for OPKG in bin/*/packages/packages/*ruby*; do
13
+        tar --to-stdout -xzf "$OPKG" ./data.tar.gz | tar tz | sed -e 's%/$%%'
14
+	done | sort -u
15
+}
16
+
17
+
18
+echo "          Staging                    Packages"
19
+diff -y <(list_staging_files) <(list_ipkg_files)
20
+