fix troublesome pages

check.pl

@@ -0,0 +1,9 @@
+# remove separators before checking for other invalid characters
+# usage: cat file perl check.pl | ruby check.rb
+
+my $SEP = "\263";
+$_ = join('', <STDIN>);
+s/\o{347}/c/g;
+s/\o{222}/'/g;
+s/\o{231}//g;
+print join('<<<<gs>>>>',split($SEP, $_));

check.rb

@@ -0,0 +1,13 @@
+# see if ruby will complain reading input
+# usage: cat file | perl check.rb | ruby check.rb
+
+Encoding.default_external = Encoding::UTF_8
+i = 0
+while x = $stdin.gets
+  begin
+    i += 1
+    x.scan(/e/)
+  rescue Exception => e
+    puts "#{i} #{x}"
+  end
+end

check.sh

@@ -0,0 +1,8 @@
+# check all remaining pages with trouble
+# sh check.sh; ls -lSr invalid
+
+cat pages/* | \
+  jq -r 'select(.trouble)|.page' | \
+  while read i; do
+    cat trouble/$i | perl check.pl | ruby check.rb > invalid/$i
+  done

json.pl

@@ -0,0 +1,15 @@
+# extract specific fields from legacy wiki format files
+# usage: cat PageName | perl json.pl date rev text
+
+my $SEP = "\263";
+$_ = join('', <STDIN>);
+s/\o{347}/c/g;
+s/\o{222}/'/g;
+s/\o{231}//g;
+%fields = split $SEP, $_;
+@selected = ();
+for (@ARGV) {
+  push @selected, $_;
+  push @selected, $fields{$_};
+}
+print join '<<<<gs>>>>', @selected;

json.rb

@@ -5,17 +5,39 @@
 
 Encoding.default_external = Encoding::UTF_8
 
-def get file
-  raw = `cat #{file} | ./a.out`
-  it = Hash[raw.split(/<<<<gs>>>>/).each_slice(2).to_a]
-  {date:it['date'], text:it['text']}
+@text = @copy = @trouble = 0
+
+def sep text
+  Hash[text.split(/<<<<gs>>>>/).each_slice(2).to_a]
+end
+
+def get page
+  it = sep `cat trouble/#{page} | perl json.pl date text rev`
+  print '.'
+  @text += 1
+  {date:it['date'], text:it['text'], rev:it['rev'], page:page}
 rescue Exception => e
-  {}
+  begin
+    it = sep `cat trouble/#{page} | perl json.pl date copy rev`
+    raise 'missing' if it['copy'].empty?
+    print ','
+    @copy += 1
+    {date:it['date'], text:it['copy'], rev:it['rev'], page:page, copy: true}
+  rescue Exception => e
+    print 'x'
+    @trouble += 1
+    {page:page, trouble:true}
+  end
 end
 
-Dir.glob('wiki.wdb/*') do |file|
-  File.open(file.gsub(/wiki.wdb/, 'static/pages'),'w') do |output|
-    output.puts JSON.pretty_generate(get(file))
+Dir.glob('trouble/*') do |file|
+  page = file.gsub(/trouble\//, '')
+  File.open("pages/#{page}",'w') do |output|
+    output.puts JSON.pretty_generate(get(page))
   end
-  print '.'
-end
+end
+
+puts
+puts "#{@text} text ok"
+puts "#{@copy} copy ok"
+puts "#{@trouble} with trouble"