diff --git a/2014-Group5/Java/Readme.txt b/2014-Group5/Java/Readme.txt new file mode 100644 index 0000000..22e3eec --- /dev/null +++ b/2014-Group5/Java/Readme.txt @@ -0,0 +1,13 @@ +The src document applies the source codes for the coursework such as analysis the raw twitter data and the sentiment analysis platforms, Stanford and Lingpipe. + +-classifier.txt +This file contains the sentiment classifier collection which will be used to do the Lingpipe sentiment process. + +-SentimentClassfier.java +This java source code is used to do the Lingpipe sentiment process and get the result by positive, neutral and negative. + +-testlingpipe.java +This java source file used to find the each companies data, user friends, followers and sentiment analysis for each twitter, then summary and create the results file. The sentiment analysis process is using Lingpipe. + +-test +This java source file is used the Stanford¡¯s Deeply Moving platform for sentiment analysis to testing the data to compare with the Lingpipe results. diff --git a/2014-Group5/Java/SentimentClassifier.java b/2014-Group5/Java/SentimentClassifier.java new file mode 100644 index 0000000..f230c90 --- /dev/null +++ b/2014-Group5/Java/SentimentClassifier.java @@ -0,0 +1,27 @@ +package groupproject; + +import java.io.File; +import java.io.IOException; + +import com.aliasi.classify.ConditionalClassification; +import com.aliasi.classify.LMClassifier; +import com.aliasi.util.AbstractExternalizable; + + public class SentimentClassifier { + String[] categories; + LMClassifier c; + public SentimentClassifier() { + try { + c= (LMClassifier) AbstractExternalizable.readObject(new File("C:\\Users\\Elvis\\Desktop\\group project\\src\\groupproject\\classifier.txt")); + categories = c.categories(); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + public String classify(String text) { + ConditionalClassification classification = c.classify(text); + return classification.bestCategory(); + } + } \ No newline at end of file diff --git a/2014-Group5/Java/Test.java b/2014-Group5/Java/Test.java new file mode 100644 index 0000000..59cf4a9 --- /dev/null +++ b/2014-Group5/Java/Test.java @@ -0,0 +1,281 @@ +package groupproject; + + + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.io.*; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import twitter4j.Status; +import twitter4j.TwitterException; +import twitter4j.TwitterObjectFactory; +import edu.stanford.nlp.ling.*; +import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations; +import edu.stanford.nlp.pipeline.*; +import edu.stanford.nlp.sentiment.SentimentCoreAnnotations; +import edu.stanford.nlp.trees.*; +import edu.stanford.nlp.util.*; + +public class Test { + + //get the date of each tweet + public String getdate(Status s){ + String d = null; + Date dateObj = s.getCreatedAt(); + d = String.format("%d/%d",dateObj.getDate(),(dateObj.getMonth()+1));//get month is beginning from 0. + return d; + } + + //get each tweet + public String gettweets(Status s){ + String tweet = null; + tweet = s.getText(); + return tweet; + } + + //get English tweets + public boolean getlang(Status s){ + boolean lang = false; + if(s.getLang().equals("en")) + lang = true; + return lang; + } + + //find ibm,intel and ge company, give them a number + public int norcompany(Status s){ + int num = 0; + String tweet = gettweets(s).toLowerCase();//transfer all the tweet into lower letter + if(tweet.contains(" ibm ")||tweet.contains(" ibm.")||tweet.contains(" ibm,")) + num = 1; + else if(tweet.contains(" intel ")||tweet.contains(" intel.")||tweet.contains(" intel,")) + num = 2; + else if(tweet.contains("general electric")) + num =3; + return num; + } + + //search the IBM with stock symbol + public boolean stockIBM(Status s){ + boolean stock = false; + String tweet = gettweets(s).toLowerCase(); + if(tweet.contains("$ibm")) + stock = true; + return stock; + } + + //search the Intel with stock symbol + public boolean stockIntel(Status s){ + boolean stock = false; + String tweet = gettweets(s).toLowerCase(); + if(tweet.contains("$intc")) + stock = true; + return stock; + } + + //search the GE with stock symbol + public boolean stockGE(Status s){ + boolean stock = false; + String tweet = gettweets(s).toLowerCase(); + if(tweet.contains("$ge")) + stock = true; + return stock; + } + + static enum Output { + PENNTREES, VECTORS, ROOT, PROBABILITIES + } + + public int sentiment(Status s){ + int score = -1; + // We initialize the StanfordCoreNLP class to process standard text input. + Properties props = new Properties(); + props.setProperty("ssplit.eolonly", "true"); + props.setProperty("annotators", "tokenize, ssplit, parse, sentiment"); + + // Each line will be treated as a single sentence. + List outputFormats = Arrays.asList(new Output[] { Output.ROOT }); + StanfordCoreNLP pipeline = new StanfordCoreNLP(props); + + // We test the systems speed +// System.out.println(new Date( ) + "\n"); +// for (int k = 0; k < 100; k++) { + int label = -1; + //String line = "I was very fond of that movie. It had perfect speed and good characters."; + //regex to replace all the html links and hash-tag # + String line = s.getText(); + String regEX = "([http|https]+[://]+[0-9A-Za-z:/[-]_#[?][=][.][&]]*)|#"; + Pattern pat = Pattern.compile(regEX); + Matcher mat = pat.matcher(line); + String newline = mat.replaceAll(""); + newline = newline.trim(); + if (newline.length() > 0) { + Annotation annotation = pipeline.process(line); + for (CoreMap sentence : annotation.get(CoreAnnotations.SentencesAnnotation.class)) { + Tree tree = sentence.get(SentimentCoreAnnotations.AnnotatedTree.class); + for (Output output : outputFormats) { + switch (output) { + case ROOT: { + CoreLabel cl = (CoreLabel) tree.label(); + cl.setValue(Integer.toString(RNNCoreAnnotations.getPredictedClass(tree))); + label = Integer.parseInt(cl.value()); // Takes values 0=Very negative 1=negative, 2=neutral, 3=positive, and 4=very positive + //System.out.println("Label is: " + label); + break; + } + default: + break; + } + } + } + } + + // The result is now stored in label. +// } + score = label; + return score; + } + + + public void analysetweets(String s, String a){ + //hashmap store the results + HashMap> results = new HashMap>();// keys=days+companynumber, value=[companynumber, very positive tweets... very negative tweets] + Status tline = null; + String filename = s; + String dest = a; + File infile = new File(filename); + File outfile = new File(dest); + try{ + //read file data + BufferedReader reader = new BufferedReader(new FileReader(infile)); + BufferedWriter writer = new BufferedWriter(new FileWriter(outfile)); + String line = null; + int count = 0; + while((line=reader.readLine()) != null){ + System.out.println(count++); +// if (line.trim().length() == 0) continue; + try { + tline = TwitterObjectFactory.createStatus(line);//create a new status for analysis + } catch(Exception e) { continue; } + //first, judge the language, we only need the english tweets. + if(getlang(tline)){ + String date = getdate(tline); + int comnum = norcompany(tline); + String key = date+", "+String.valueOf(comnum)+", ";//combine the date and company number together as the key of the hashmap + //if there not exist a arraylist with the key + if(!results.containsKey(key)){ + ArrayList IBMlist = new ArrayList();//new arrylist for IBM + for(int i=0;i<10;i++){ + IBMlist.add(i, 0); + } + ArrayList Intellist = new ArrayList(); + for(int i=0;i<10;i++){ + Intellist.add(i, 0); + } + ArrayList GElist = new ArrayList(); + for(int i=0;i<10;i++){ + GElist.add(i, 0); + } + //organize the tweets into different arraylist + switch(norcompany(tline)){ + //IBM = 1 + case 1: + if(stockIBM(tline)){ + // Takes values 0=$verynegative, 1=$negative, 2=$neutral, 3=$positive, 4=$verypositive, 5=verynegative, 6=negative, 7=neutral, 8=positive, 9=verypositive + IBMlist.set(sentiment(tline),1); + }else{ + IBMlist.set(sentiment(tline)+5,1); + } + results.put(key, IBMlist); + //Intel = 2 + case 2: + if(stockIntel(tline)){ + // Takes values 0=$verynegative, 1=$negative, 2=$neutral, 3=$positive, 4=$verypositive, 5=verynegative, 6=negative, 7=neutral, 8=positive, 9=verypositive + Intellist.set(sentiment(tline),1); + }else{ + Intellist.set(sentiment(tline)+5,1); + } + results.put(key, Intellist); + //GE = 3 + case 3: + if(stockGE(tline)){ + // Takes values 0=$verynegative, 1=$negative, 2=$neutral, 3=$positive, 4=$verypositive, 5=verynegative, 6=negative, 7=neutral, 8=positive, 9=verypositive + GElist.set(sentiment(tline),1); + }else{ + GElist.set(sentiment(tline)+5,1); + } + results.put(key, GElist); + } + } + // if the key has existed + else{ + switch(norcompany(tline)){ + //reset the value of IBM =1 + case 1: + ArrayList IBMlist = results.get(key); + if(stockIBM(tline)){ + IBMlist.set(sentiment(tline),IBMlist.get(sentiment(tline))+1); + }else{ + IBMlist.set(sentiment(tline)+5,IBMlist.get(sentiment(tline)+5)+1); + } + results.put(key, IBMlist); + //reset the value of Intel =2 + case 2: + ArrayList Intellist = results.get(key); + if(stockIntel(tline)){ + Intellist.set(sentiment(tline),Intellist.get(sentiment(tline))+1); + }else{ + Intellist.set(sentiment(tline)+5,Intellist.get(sentiment(tline)+5)+1); + } + results.put(key, Intellist); + //reset the value of Intel =2 + case 3: + ArrayList GElist = results.get(key); + if(stockGE(tline)){ + GElist.set(sentiment(tline),GElist.get(sentiment(tline))+1); + }else{ + GElist.set(sentiment(tline)+5,GElist.get(sentiment(tline)+5)+1); + } + results.put(key, GElist); + } + } + } + } + //write the result into the new file + Iterator iterator = results.keySet().iterator(); + String l = null; + String fr = null; + while (iterator.hasNext()) { + l = iterator.next(); + fr = l+results.get(l); + writer.write(fr); + writer.newLine(); + writer.flush(); + } + reader.close(); + writer.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + + public static void main(String[] args){ + Test test = new Test(); + String in = "C:\\Users\\Elvis\\Desktop\\group project\\data\\outputData\\testsl.txt"; + String out = "C:\\Users\\Elvis\\Desktop\\group project\\data\\stanford_out.txt"; + test.analysetweets(in, out); + } + +} diff --git a/2014-Group5/Java/classifier.txt b/2014-Group5/Java/classifier.txt new file mode 100644 index 0000000..b2bb651 Binary files /dev/null and b/2014-Group5/Java/classifier.txt differ diff --git a/2014-Group5/Java/testlingpipe.java b/2014-Group5/Java/testlingpipe.java new file mode 100644 index 0000000..27780a5 --- /dev/null +++ b/2014-Group5/Java/testlingpipe.java @@ -0,0 +1,346 @@ +package groupproject; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.io.*; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import twitter4j.Status; +import twitter4j.TwitterException; +import twitter4j.TwitterObjectFactory; +import edu.stanford.nlp.ling.*; +import edu.stanford.nlp.neural.rnn.RNNCoreAnnotations; +import edu.stanford.nlp.pipeline.*; +import edu.stanford.nlp.sentiment.SentimentCoreAnnotations; +import edu.stanford.nlp.trees.*; +import edu.stanford.nlp.util.*; + +public class testlingpipe{ + + //get the date of each tweet + public String getdate(Status s){ + String d = null; + Date dateObj = s.getCreatedAt(); + d = String.format("%d/%d",dateObj.getDate(),(dateObj.getMonth()+1));//get month is beginning from 0. + return d; + } + + //get each tweet + public String gettweets(Status s){ + String tweet = null; + tweet = s.getText(); + return tweet; + } + + //get English tweets + public boolean getlang(Status s){ + boolean lang = false; +// boolean l = false; + try{ + if(s.getLang().equals("en")){ + lang = true; + } + }catch(Exception e){ + + } + return lang; + } + + // Does tweet contain url? + public boolean containsurl(Status s){ + if(s.getURLEntities().length > 0) { + return true; + } else { + return false; + } + } + + // get number of followers + public int getfollowercount(Status s){ + return s.getUser().getFollowersCount(); + } + + // get number of friends + public int getfriendcount(Status s){ + return s.getUser().getFriendsCount(); + } + + + //find ibm,intel and ge company, give them a number + public int norcompany(Status s){ + int num = 0; + String tweet = gettweets(s).toLowerCase();//transfer all the tweet into lower letter + if(tweet.contains(" ibm ")||tweet.contains(" ibm.")||tweet.contains(" ibm,")) + num = 1; + else if(tweet.contains(" intel ")||tweet.contains(" intel.")||tweet.contains(" intel,")) + num = 2; + else if(tweet.contains("general electric")) + num =3; + return num; + } + + //search the IBM with stock symbol + public boolean stockIBM(Status s){ + boolean stock = false; + String tweet = gettweets(s).toLowerCase(); + if(tweet.contains("$ibm")) + stock = true; + return stock; + } + + //search the Intel with stock symbol + public boolean stockIntel(Status s){ + boolean stock = false; + String tweet = gettweets(s).toLowerCase(); + if(tweet.contains("$intc")) + stock = true; + return stock; + } + + //search the GE with stock symbol + public boolean stockGE(Status s){ + boolean stock = false; + String tweet = gettweets(s).toLowerCase(); + if(tweet.contains("$ge")) + stock = true; + return stock; + } + + static enum Output { + PENNTREES, VECTORS, ROOT, PROBABILITIES + } + + public int sentiment(Status s){ + int score = -1; + SentimentClassifier sentClassifier; + sentClassifier = new SentimentClassifier(); + String line = s.getText(); + String regEX = "([http|https]+[://]+[0-9A-Za-z:/[-]_#[?][=][.][&]]*)|#"; + Pattern pat = Pattern.compile(regEX); + Matcher mat = pat.matcher(line); + String newline = mat.replaceAll(""); + newline = newline.trim(); + String sent = sentClassifier.classify(newline); + if(sent.equalsIgnoreCase("neg")){ + score = 0; + }else if(sent.equalsIgnoreCase("neu")){ + score = 1; + }else if(sent.equalsIgnoreCase("pos")){ + score = 2; + } + return score; + } + + + public void analysetweets(String s, String a){ + //hashmap store the results + HashMap> results = new HashMap>();// keys=days+companynumber, value=[companynumber, very positive tweets... very negative tweets] + Status tline = null; + String filename = s; + String dest = a; + File infile = new File(filename); + File outfile = new File(dest); + try{ + //read file data + BufferedReader reader = new BufferedReader(new FileReader(infile)); + BufferedWriter writer = new BufferedWriter(new FileWriter(outfile)); + String line = null; + int count = 0; + while((line=reader.readLine()) != null){ + System.out.println("Reading line: " + count++); +// if (line.trim().length() == 0) continue; + try { + tline = TwitterObjectFactory.createStatus(line);//create a new status for analysis + } catch(Exception e) { continue; } + + //System.out.println(" tweet: " + tline.getText()); + //System.out.println(" followers count: " + getfollowercount(tline)); + + //first, judge the language, we only need the english tweets. + if(getlang(tline)){ + String date = getdate(tline); + int comnum = norcompany(tline); + String key = date+", "+String.valueOf(comnum)+", ";//combine the date and company number together as the key of the hashmap + //if there not exist a arraylist with the key + if(!results.containsKey(key)){ + ArrayList IBMlist = new ArrayList();//new arrylist for IBM + for(int i=0;i<72;i++){ + IBMlist.add(i, 0); + } + ArrayList Intellist = new ArrayList(); + for(int i=0;i<72;i++){ + Intellist.add(i, 0); + } + ArrayList GElist = new ArrayList(); + for(int i=0;i<72;i++){ + GElist.add(i, 0); + } + + int friendsCount = getfriendcount(tline); + int followersCount = getfollowercount(tline); + boolean tweetHasUrl = containsurl(tline); + int urlOffset = 0; + if (tweetHasUrl == true) { + urlOffset = 6; + } + + for (int friendsOffset=0; friendsOffset <= 60; friendsOffset = friendsOffset + 12) { + int friendsMultiplicationFactor = 1; + if (friendsOffset == 12) { + friendsMultiplicationFactor = friendsCount; + } + if (friendsOffset == 24) { + friendsMultiplicationFactor = (int) (Math.log(friendsCount+1) / Math.log(2)); + } + if (friendsOffset == 36) { + friendsMultiplicationFactor = 1; + } + if (friendsOffset == 48) { + friendsMultiplicationFactor = followersCount; + } + if (friendsOffset == 60) { + friendsMultiplicationFactor = (int) (Math.log(followersCount+1) / Math.log(2)); + } + int currentOffset = friendsOffset + urlOffset; + + + //organize the tweets into different arraylist + switch(norcompany(tline)){ + //IBM = 1 + case 1: + if(stockIBM(tline)){ + // Takes values 0=negative, 1=neutral and 2=positive + IBMlist.set(sentiment(tline)+currentOffset,friendsMultiplicationFactor); + }else{ + IBMlist.set(sentiment(tline)+currentOffset+3,friendsMultiplicationFactor); + } + results.put(key, IBMlist); + //Intel = 2 + case 2: + if(stockIntel(tline)){ + // Takes values 0=negative, 1=neutral and 2=positive + Intellist.set(sentiment(tline)+currentOffset,friendsMultiplicationFactor); + }else{ + Intellist.set(sentiment(tline)+currentOffset+3,friendsMultiplicationFactor); + } + results.put(key, Intellist); + //GE = 3 + case 3: + if(stockGE(tline)){ + // Takes values 0=negative, 1=neutral and 2=positive + GElist.set(sentiment(tline)+currentOffset,friendsMultiplicationFactor); + }else{ + GElist.set(sentiment(tline)+currentOffset+3,friendsMultiplicationFactor); + } + results.put(key, GElist); + } + } + } + // if the key has existed + else{ + int friendsCount = getfriendcount(tline); + int followersCount = getfollowercount(tline); + + boolean tweetHasUrl = containsurl(tline); + int urlOffset = 0; + if (tweetHasUrl == true) { + urlOffset = 6; + } + + for (int friendsOffset=0; friendsOffset <= 60; friendsOffset = friendsOffset + 12) { + int friendsMultiplicationFactor = 1; + if (friendsOffset == 12) { + friendsMultiplicationFactor = friendsCount; + } + if (friendsOffset == 24) { + friendsMultiplicationFactor = (int) (Math.log(friendsCount+1) / Math.log(2)); + } + if (friendsOffset == 36) { + friendsMultiplicationFactor = 1; + } + if (friendsOffset == 48) { + friendsMultiplicationFactor = followersCount; + } + if (friendsOffset == 60) { + friendsMultiplicationFactor = (int) (Math.log(followersCount+1) / Math.log(2)); + } + int currentOffset = friendsOffset + urlOffset; + + + switch(norcompany(tline)){ + //reset the value of IBM =1 + case 1: + ArrayList IBMlist = results.get(key); + if(stockIBM(tline)){ + IBMlist.set(sentiment(tline)+currentOffset,IBMlist.get(sentiment(tline)+currentOffset)+friendsMultiplicationFactor); + }else{ + IBMlist.set(sentiment(tline)+currentOffset+3,IBMlist.get(sentiment(tline)+currentOffset+3)+friendsMultiplicationFactor); + } + results.put(key, IBMlist); + //reset the value of Intel =2 + case 2: + ArrayList Intellist = results.get(key); + if(stockIntel(tline)){ + Intellist.set(sentiment(tline)+currentOffset,Intellist.get(sentiment(tline)+currentOffset)+friendsMultiplicationFactor); + }else{ + Intellist.set(sentiment(tline)+currentOffset+3,Intellist.get(sentiment(tline)+currentOffset+3)+friendsMultiplicationFactor); + } + results.put(key, Intellist); + //reset the value of Intel =2 + case 3: + ArrayList GElist = results.get(key); + if(stockGE(tline)){ + GElist.set(sentiment(tline)+currentOffset,GElist.get(sentiment(tline)+currentOffset)+friendsMultiplicationFactor); + }else{ + GElist.set(sentiment(tline)+currentOffset+3,GElist.get(sentiment(tline)+currentOffset+3)+friendsMultiplicationFactor); + } + results.put(key, GElist); + } + } + } + } + } + //write the result into the new file + Iterator iterator = results.keySet().iterator(); + String l = null; + String fr = null; + while (iterator.hasNext()) { + l = iterator.next(); + fr = l+results.get(l); + writer.write(fr); + writer.newLine(); + writer.flush(); + } + reader.close(); + writer.close(); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + + public static void main(String[] args){ + testlingpipe test = new testlingpipe(); + + // Xuyang's paths + String in = "C:\\Users\\Elvis\\Desktop\\group project\\data\\outputData\\store11.txt"; + String out = "C:\\Users\\Elvis\\Desktop\\group project\\data\\store11_out.txt"; + + // Julian's paths + //String in = "/home/julian/UCL/Courses/Information Retrieval/Group Project/Xuyang Code/data/asd.txt"; + //String out = "/home/julian/UCL/Courses/Information Retrieval/Group Project/Xuyang Code/data/new_out.txt"; + test.analysetweets(in, out); + } + +} diff --git a/2014-Group5/Matlab/BasicLassoModel.m b/2014-Group5/Matlab/BasicLassoModel.m new file mode 100644 index 0000000..4e4c44e --- /dev/null +++ b/2014-Group5/Matlab/BasicLassoModel.m @@ -0,0 +1,203 @@ +% We build the benchmark linear lasso regression model here. This is the same +% as the one proposed by Mao et al, but with lasso penality. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +stockname = 'GE'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Extract total Tweets count related to the stock +TotalTweetsCount = sum(TwitterFeatures(:, [1,2,3,7,8,9])')'; + +% Normalize all features to have zero mean and standard deviation one +TotalTweetsCount = zscore(TotalTweetsCount); +StockFeatures = zscore(StockFeatures); + + +% Build linear model separately for each dependent variable +PredictedTables = zeros(TestDataSize, 4); +TwitterDaysUsed = zeros(TestDataSize, 4); +for j = 1:4 + % For each test sample, we use leave-one-out cross-validation to pick + % the optimal m (0-3) and n (0-3) and use the best model to predict the + % next future value. + for i = (TotalDataSize-TestDataSize+1):TotalDataSize + i + % We use leave-one-out cross-validation on the previous 10 samples to find the optimal m and n + CrossValidationResults=zeros(3,4); + + OptimalSquaredError = 10000000; + Optimalm = 1; + Optimaln = 0; + % We vary the number of previous stock feature inputs + for m=1:3 + % We vary the number of previous Twitter feature inputs + for n=0:3 + for l=(i-10):i + TrainingInput = []; + ValidationInput = []; + + for k=1:m + TrainingInput = [TrainingInput, StockFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, StockFeatures((l-k),:)]; + end; + + for k=1:n + TrainingInput = [TrainingInput, TotalTweetsCount((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, TotalTweetsCount((l-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(l-1),j); + ValidationOutput = StockFeatures(l,j); + + % TestInput = [StockFeatures(l,:), TotalTweetsCount(l,:)]; + CrossValidationPrediction = LinearLasso(TrainingInput, ValidationInput, TrainingOutput); + + %CrossValidationResults(l-(i-10)+1,m,n+1) = (ValidationOutput-CrossValidationPrediction).^2; + CrossValidationResults(m,n+1) = CrossValidationResults(m,n+1) + (ValidationOutput-CrossValidationPrediction).^2; + end; + + if (OptimalSquaredError > CrossValidationResults(m,n+1)) + OptimalSquaredError = CrossValidationResults(m,n+1); + Optimalm = m; + Optimaln = n; + end; + end; + end; + + % We use the optimal n and m found from cross-validation to build a + % new linear regression model based on all the previous data + % samples, and use it to predict the next sample. + TrainingInput = []; + TestInput = []; + + for k=1:Optimalm + TrainingInput = [TrainingInput, StockFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, StockFeatures((l-k),:)]; + end; + + for k=1:Optimaln + TrainingInput = [TrainingInput, TotalTweetsCount((4-k):(i-k-1),:)]; + TestInput = [TestInput, TotalTweetsCount((i-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(i-1),j); + + PredictedTables(i-(TotalDataSize-TestDataSize),j) = LinearLasso(TrainingInput, TestInput, TrainingOutput); + + TwitterDaysUsed(i-(TotalDataSize-TestDataSize),j) = Optimaln; + end; +end; + +% Plot the results. This is good for the report and will make it easy to +% interpret the model performance. + +h=figure; +%suptitle('Predictions for General Electric', 'FontType', 'Times'); +subplot(2,2,1); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,1), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,1), 'LineWidth', 2); +ylabel('Volume (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,2); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,2), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,2), 'LineWidth', 2); +ylabel('Closing Price (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,3); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,3), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,3), 'LineWidth', 2); +ylabel('Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,4); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,4), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,4), 'LineWidth', 2); +ylabel('Abs Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Predictions for ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('Predictions_', stockname, '')); +saveas(h,strcat('Predictions_', stockname, ''),'png'); + +% Calculate mean absolute error, squared error and accuracy (positive versus negative) +fprintf('Trading Volume (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1))), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,1) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + + +fprintf('Closing Price (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2))), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,2) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3))), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,3) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Abs Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4))), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,4) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); diff --git a/2014-Group5/Matlab/BasicLinearModel.m b/2014-Group5/Matlab/BasicLinearModel.m new file mode 100644 index 0000000..d6a6e31 --- /dev/null +++ b/2014-Group5/Matlab/BasicLinearModel.m @@ -0,0 +1,223 @@ +% We build the baselinle / benchmark linear model here. This is the same +% as the one proposed by Mao et al. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +stockname = 'Intel'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Extract total Tweets count related to the stock +TotalTweetsCount = sum(TwitterFeatures(:, [1,2,3,7,8,9])')'; + +% Normalize all features to have zero mean and standard deviation one +TotalTweetsCount = zscore(TotalTweetsCount); +StockFeatures = zscore(StockFeatures); + +figure; +subplot(2,1,1); +hold on; +bar(1:50,zscore(TotalTweetsCount)); +plot(1:50, zscore(StockFeatures(:,1)), 'r'); +legend('Intel Tweets Count', 'Trading Volume'); +xlabel('Day'); + +subplot(2,1,2); +hold on; +bar(1:50,zscore(TotalTweetsCount)); +plot(1:50, zscore(StockFeatures(:,3)), 'r'); +legend('Intel Tweets Count', 'Daily Price Change'); +xlabel('Day'); + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(stockname, 'FontSize', 16, 'FontWeight', 'Bold'); + + + +% Build linear model separately for each dependent variable +PredictedTables = zeros(TestDataSize, 4); +TwitterDaysUsed = zeros(TestDataSize, 4); +for j = 1:4 + % For each test sample, we use leave-one-out cross-validation to pick + % the optimal m (0-3) and n (0-3) and use the best model to predict the + % next future value. + for i = (TotalDataSize-TestDataSize+1):TotalDataSize + i + % We use leave-one-out cross-validation on the previous 10 samples to find the optimal m and n + CrossValidationResults=zeros(3,4); + + OptimalSquaredError = 10000000; + Optimalm = 1; + Optimaln = 0; + % We vary the number of previous stock feature inputs + for m=1:3 + % We vary the number of previous Twitter feature inputs + for n=0:3 + for l=(i-10):i + TrainingInput = []; + ValidationInput = []; + + for k=1:m + TrainingInput = [TrainingInput, StockFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, StockFeatures((l-k),:)]; + end; + + for k=1:n + TrainingInput = [TrainingInput, TotalTweetsCount((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, TotalTweetsCount((l-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(l-1),j); + ValidationOutput = StockFeatures(l,j); + + % TestInput = [StockFeatures(l,:), TotalTweetsCount(l,:)]; + CrossValidationPrediction = LinearRR(TrainingInput, ValidationInput, TrainingOutput,1^(-10)); + + %CrossValidationResults(l-(i-10)+1,m,n+1) = (ValidationOutput-CrossValidationPrediction).^2; + CrossValidationResults(m,n+1) = CrossValidationResults(m,n+1) + (ValidationOutput-CrossValidationPrediction).^2; + end; + + if (OptimalSquaredError > CrossValidationResults(m,n+1)) + OptimalSquaredError = CrossValidationResults(m,n+1); + Optimalm = m; + Optimaln = n; + end; + end; + end; + + % We use the optimal n and m found from cross-validation to build a + % new linear regression model based on all the previous data + % samples, and use it to predict the next sample. + TrainingInput = []; + TestInput = []; + + for k=1:Optimalm + TrainingInput = [TrainingInput, StockFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, StockFeatures((l-k),:)]; + end; + + for k=1:Optimaln + TrainingInput = [TrainingInput, TotalTweetsCount((4-k):(i-k-1),:)]; + TestInput = [TestInput, TotalTweetsCount((i-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(i-1),j); + + PredictedTables(i-(TotalDataSize-TestDataSize),j) = LinearRR(TrainingInput, TestInput, TrainingOutput,1^(-10)); + + TwitterDaysUsed(i-(TotalDataSize-TestDataSize),j) = Optimaln; + end; +end; + +% Plot the results. This is good for the report and will make it easy to +% interpret the model performance. + +h=figure; +%suptitle('Predictions for General Electric', 'FontType', 'Times'); +subplot(2,2,1); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,1), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,1), 'LineWidth', 2); +ylabel('Volume (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,2); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,2), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,2), 'LineWidth', 2); +ylabel('Closing Price (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,3); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,3), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,3), 'LineWidth', 2); +ylabel('Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,4); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,4), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,4), 'LineWidth', 2); +ylabel('Abs Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Predictions for ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('Predictions_', stockname, '')); +saveas(h,strcat('Predictions_', stockname, ''),'png'); + +% Calculate mean absolute error, squared error and accuracy (positive versus negative) +fprintf('Trading Volume (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1))), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,1) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + + +fprintf('Closing Price (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2))), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,2) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3))), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,3) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Abs Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4))), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,4) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); diff --git a/2014-Group5/Matlab/ConstantModel.m b/2014-Group5/Matlab/ConstantModel.m new file mode 100644 index 0000000..e4ab632 --- /dev/null +++ b/2014-Group5/Matlab/ConstantModel.m @@ -0,0 +1,81 @@ +% We build the baselinle / benchmark linear model here. This is the same +% as the one proposed by Mao et al. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +stockname = 'GE'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Normalize all features to have zero mean and standard deviation one +TotalTweetsCount = zscore(TotalTweetsCount); +StockFeatures = zscore(StockFeatures); + +% As a benchmark model we will have a constant model. That is, we predict +% the previous indicator with the current indicator. As this requires only +% a single previous data point, we will use this as our prediction. + +fprintf('Constant Model \n'); +fprintf('Trading Volume (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,1) - StockFeatures(2:end,1))), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,1) - StockFeatures(2:end,1)).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); +fprintf('Accuracy (Pos vs. Neg) %8.3f n/a \n \n', length(find(StockFeatures(1:end-1,1).*StockFeatures(2:end,1) >= 0))/(TotalDataSize-1)); + + +fprintf('Closing Price (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,2) - StockFeatures(2:end,2))), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)))); +fprintf('Squared Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,2) - StockFeatures(2:end,2)).^2), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2)); +fprintf('Accuracy (Pos vs. Neg) %8.3f n/a \n\n', length(find(StockFeatures(1:end-1,2).*StockFeatures(2:end,2) >= 0))/(TotalDataSize-1)); + +fprintf('Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,3) - StockFeatures(2:end,3))), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)))); +fprintf('Squared Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,3) - StockFeatures(2:end,3)).^2), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2)); +fprintf('Accuracy (Pos vs. Neg) %8.3f n/a \n\n', length(find(StockFeatures(1:end-1,3).*StockFeatures(2:end,3) >= 0))/(TotalDataSize-1)); + +fprintf('Abs Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,4) - StockFeatures(2:end,4))), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)))); +fprintf('Squared Error %8.3f %8.3f \n', mean(abs(StockFeatures(1:end-1,4) - StockFeatures(2:end,4)).^2), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2)); +fprintf('Accuracy (Pos vs. Neg) %8.3f n/a \n\n', length(find(StockFeatures(1:end-1,4).*StockFeatures(2:end,4) >= 0))/(TotalDataSize-1)); + diff --git a/2014-Group5/Matlab/ExtrapolateStockData.m b/2014-Group5/Matlab/ExtrapolateStockData.m new file mode 100644 index 0000000..170aa44 --- /dev/null +++ b/2014-Group5/Matlab/ExtrapolateStockData.m @@ -0,0 +1,72 @@ +% We load in file CSV file. It has columns Company Index, Month, Day, +% Features. + +stockname = 'GE'; +stockfilename = 'StockPrices/GE.csv'; + +A = importdata(stockfilename,','); + +StockPrices = A.data(:,1:5); +StockDates = A.textdata(2:end,1); +StockDays = zeros(size(StockDates, 1), 1); + +RealStockDays=[]; + +for i=1:size(StockDates,1); + DifferenceBetweenDays = datenum(StockDates(i), 'yyyy-mm-dd') - datenum('01/12/2014'); + StockDays(i) = DifferenceBetweenDays; + + if (i > 2) && (StockDays(i-1) - StockDays(i) == 1) + RealStockDays = [RealStockDays, StockDays(i)]; + end; +end; + +% Sort by day +[~, OrderedIndices] = sort(StockDays); +StockDays = StockDays(OrderedIndices); +StockPrices = StockPrices(OrderedIndices,:); + +% Build AR model based on first 4 points +%ModelOne = ar(StockPrices(1:4,1),2); +%ModelTwo = ar(StockPrices(1:4,2),2); +%ModelThree = ar(StockPrices(1:4,3),2); +%ModelFour = ar(StockPrices(1:4,4),2); +%ModelFive = ar(StockPrices(1:4,5),2); +%coeff = polydata(m); + +% Extrapolate whenever data is missing +ExtrapolationCompleted = false; +while (ExtrapolationCompleted==false) + ExtrapolationCompleted = true; + for i=3:size(StockDays,1); + i + if round(StockDays(i) - StockDays(i-1)) > 1 + ExtrapolationCompleted = false; + + % Extrapolate one day forward + NewRow = zeros(1,5); + for j=1:5; + model = ar(iddata(StockPrices(1:i,j), [], 1),2); + predictions = predict(model, iddata(StockPrices(1:i,j), [], 1)); + NewRow(j) = predictions.OutputData(end); + end; + + % Substitute in extrapolation + StockDays = [StockDays(1:(i-1)); StockDays(i-1)+1; StockDays(i:end)]; + StockPrices = [StockPrices(1:(i-1),:); NewRow; StockPrices(i:end,:)]; + + break; + end; + end; +end; + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = zeros(size(StockPrices,1)-1, size(StockPrices,2)-1); +StockFeatures(:,1) = StockPrices(2:end,5); +StockFeatures(:,2) = StockPrices(2:end,4); +StockFeatures(:,3) = diff(StockPrices(:,4)); +StockFeatures(:,4) = abs(diff(StockPrices(:,4))); + +% Save stock features +save(strcat(stockname, 'Features.mat'),'StockFeatures'); + diff --git a/2014-Group5/Matlab/FeatureAnalysis.m b/2014-Group5/Matlab/FeatureAnalysis.m new file mode 100644 index 0000000..24df053 --- /dev/null +++ b/2014-Group5/Matlab/FeatureAnalysis.m @@ -0,0 +1,158 @@ +% We build the basic linear model here + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +stockname = 'Intel'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Extract total Tweets count related to the stock +TotalTweetsCount = sum(TwitterFeatures(:, [1,2,3,7,8,9])')'; + +% Calculate correlation coefficient statistics w.r.t. tweet counts for +% entire data set +TweetCountCorrCoefficient = zeros(size(StockFeatures,2), 2); +for j = 1:size(StockFeatures,2) + [r,p] = corrcoef(zscore(TotalTweetsCount), zscore(StockFeatures(:,j))); + TweetCountCorrCoefficient(j,1) = r(1,2); + TweetCountCorrCoefficient(j,2) = p(1,2); +end; + +TweetCountCorrCoefficient + +% Calculate correlation coefficient statistics w.r.t. all features for +% training data +AllFeaturesCorrCoefficientValues = zeros(size(TwitterFeatures,2), size(StockFeatures,2)); +AllFeaturesCorrCoefficientPValues = zeros(size(TwitterFeatures,2), size(StockFeatures,2)); +for i=1:size(TwitterFeatures,2) + i + for j = 1:size(StockFeatures,2) + [r,p] = corrcoef(zscore(TwitterFeatures(1:TrainingDataSize,i)), zscore(StockFeatures(1:TrainingDataSize,j))); + AllFeaturesCorrCoefficientValues(i,j) = r(1,2); + AllFeaturesCorrCoefficientPValues(i,j) = p(1,2); + + end; +end; + +h=figure; +subplot(2,1,1); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientValues(:,1)); +ylabel('Correlation Coefficient'); +xlabel('Feature Index'); +title('Trading Volume', 'FontWeight', 'bold', 'FontSize', 15); + +subplot(2,1,2); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientPValues(:,1)); +ylabel('Correlation Coefficient P-Value'); +xlabel('Feature Index'); +savefig(h, strcat('CorrCoeff_TradingVolume_', stockname, '')); +saveas(h,strcat('CorrCoeff_TradingVolume_', stockname, ''),'png'); +close(h); + +h=figure; +subplot(2,1,1); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientValues(:,2)); +ylabel('Correlation Coefficient'); +xlabel('Feature Index'); +title('Closing Price', 'FontWeight', 'bold', 'FontSize', 15); + +subplot(2,1,2); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientPValues(:,2)); +ylabel('Correlation Coefficient P-Value'); +xlabel('Feature Index'); +savefig(h, strcat('CorrCoeff_ClosingPrice_', stockname, '')); +saveas(h,strcat('CorrCoeff_ClosingPrice_', stockname, ''),'png'); +close(h); + +h=figure; +subplot(2,1,1); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientValues(:,3)); +ylabel('Correlation Coefficient'); +xlabel('Feature Index'); +title('Daily Price Price', 'FontWeight', 'bold', 'FontSize', 15); + +subplot(2,1,2); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientPValues(:,3)); +ylabel('Correlation Coefficient P-Value'); +xlabel('Feature Index'); +savefig(h, strcat('CorrCoeff_PriceChange_', stockname, '')); +saveas(h,strcat('CorrCoeff_PriceChange_', stockname, ''),'png'); +close(h); + + +h=figure; +subplot(2,1,1); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientValues(:,4)); +ylabel('Correlation Coefficient'); +xlabel('Feature Index'); +title('Abs Daily Price Price', 'FontWeight', 'bold', 'FontSize', 15); + +subplot(2,1,2); +set(gca, 'FontSize', 13) +bar(AllFeaturesCorrCoefficientPValues(:,4)); +ylabel('Correlation Coefficient P-Value'); +xlabel('Feature Index'); +savefig(h, strcat('CorrCoeff_AbsPriceChange_', stockname, '')); +saveas(h,strcat('CorrCoeff_AbsPriceChange_', stockname, ''),'png'); +close(h); + + + +% Find significant features. As we observe from the above plots, almost no features correlate well with the +% the closing price, so we will only the subset of features which correlate +% well with closing price. +%SignificantFeaturesIndices = find(AllFeaturesCorrCoefficientPValues(:,1) < 0.05 | AllFeaturesCorrCoefficientPValues(:,3) < 0.05 | AllFeaturesCorrCoefficientPValues(:,4) < 0.05); +SignificantFeaturesIndices = find(AllFeaturesCorrCoefficientPValues(:,1) < 0.20 | AllFeaturesCorrCoefficientPValues(:,3) < 0.20 | AllFeaturesCorrCoefficientPValues(:,4) < 0.20); + +[~, SignificantFeatureIndicesSorted] = sort(abs(AllFeaturesCorrCoefficientValues(SignificantFeaturesIndices, 3))) +SignificantFeaturesIndices = SignificantFeaturesIndices(SignificantFeatureIndicesSorted); + +fprintf('Significant features at 95% confidence level are: \n'); +for i=1:length(SignificantFeaturesIndices); + fprintf(strcat(' %4.2d > ', strcat(' ', FeatureNameDescription{i}), '\n'), round(AllFeaturesCorrCoefficientValues(SignificantFeaturesIndices(i), 3)*100)/100); +end; \ No newline at end of file diff --git a/2014-Group5/Matlab/InterpolatedStockFeatures/GEFeatures.mat b/2014-Group5/Matlab/InterpolatedStockFeatures/GEFeatures.mat new file mode 100644 index 0000000..d48096b Binary files /dev/null and b/2014-Group5/Matlab/InterpolatedStockFeatures/GEFeatures.mat differ diff --git a/2014-Group5/Matlab/InterpolatedStockFeatures/IBMFeatures.mat b/2014-Group5/Matlab/InterpolatedStockFeatures/IBMFeatures.mat new file mode 100644 index 0000000..08ba9cd Binary files /dev/null and b/2014-Group5/Matlab/InterpolatedStockFeatures/IBMFeatures.mat differ diff --git a/2014-Group5/Matlab/InterpolatedStockFeatures/IntelFeatures.mat b/2014-Group5/Matlab/InterpolatedStockFeatures/IntelFeatures.mat new file mode 100644 index 0000000..6a5d6bd Binary files /dev/null and b/2014-Group5/Matlab/InterpolatedStockFeatures/IntelFeatures.mat differ diff --git a/2014-Group5/Matlab/LinearLasso.m b/2014-Group5/Matlab/LinearLasso.m new file mode 100644 index 0000000..a045277 --- /dev/null +++ b/2014-Group5/Matlab/LinearLasso.m @@ -0,0 +1,40 @@ +function [setPredLabelsAll] = LinearLasso(setTrainX, setTestX, setTrainLabelsAll) + +%addpath('glmnet_matlab'); + +% We add a constant factor to the training and test sets to account for slope. +setTrainX = [setTrainX, ones(size(setTrainX,1),1)]; +setTestX = [setTestX, ones(size(setTestX,1),1)]; + +setPredLabelsAll = zeros(size(setTestX,1), size(setTrainLabelsAll,2)); + +%[B, FitInfo] = lasso(setTrainX, setTrainLabelsAll,'CV',2, 'NumLambda', 10, 'UseParallel', true); +%[~, i] = find(FitInfo.Lambda == FitInfo.LambdaMinMSE); + +%options = glmnetSet(); +%options.nlambda = 10; +%options.standardize = true; +%options.lambda=0:0.1:1; +%options.alpha = 1; + +%'nlambda', '10', 'standardize', 'false', 'lambda', 0:0.1:1); + +%cvfit = cvglmnet(setTrainX, setTrainLabelsAll, 'gaussian', options, 'mse', 5); +cvfit = cvglmnet(setTrainX, setTrainLabelsAll, 'gaussian', ['alpha', 1, 'nlambda', 10]); +%w = cvglmnetPredict(cvfit,[],cvfit.lambda_min,'coef'); +%cvfit.lambda_min +%w = B(:,i); +for (i = 1:size(setTestX, 1)) + %setPredLabelsAll(:,i) = glmnetPredict(cvfit,setTestX,[0.01,0.005]') + setPredLabelsAll(:,i) = cvglmnetPredict(cvfit,setTestX,'lambda_min'); + + %setPredLabelsAll(:,i) + %setPredLabelsAll(:,i) = setTestX*w; + + + % We calculate w asterisk, i.e. the w in y = wx + % w_asterisk = (setTrainX'*setTrainX + gamma * size(setTrainX,1) * eye(size(setTrainX, 2))) \ (setTrainX' * setTrainLabelsAll(:,i)); + + % We obtain the predictions + %setPredLabelsAll(:,i) = setTestX*w_asterisk; +end diff --git a/2014-Group5/Matlab/LinearRR.m b/2014-Group5/Matlab/LinearRR.m new file mode 100644 index 0000000..28ad55c --- /dev/null +++ b/2014-Group5/Matlab/LinearRR.m @@ -0,0 +1,22 @@ +function [setPredLabelsAll] = LinearRR(setTrainX, setTestX, setTrainLabelsAll, inGamma) + +% We use this piece of code for finding the baseline: +%if (1==1) +% setPredLabelsAll = repmat(mean(setTrainLabelsAll), size(setTestX,1),1); +% return; +%end; + +% We add a constant factor to the training and test sets to account for slope. +setTrainX = [setTrainX, ones(size(setTrainX,1),1)]; +setTestX = [setTestX, ones(size(setTestX,1),1)]; + +setPredLabelsAll = zeros(size(setTestX,1), size(setTrainLabelsAll,2)); +gamma = inGamma; + +for (i = 1:size(setTestX, 1)) + % We calculate w asterisk, i.e. the w in y = wx + w_asterisk = (setTrainX'*setTrainX + gamma * size(setTrainX,1) * eye(size(setTrainX, 2))) \ (setTrainX' * setTrainLabelsAll(:,i)); + + % We obtain the predictions + setPredLabelsAll(:,i) = setTestX*w_asterisk; +end diff --git a/2014-Group5/Matlab/PolynomialFeatureMap.m b/2014-Group5/Matlab/PolynomialFeatureMap.m new file mode 100644 index 0000000..fe348b8 --- /dev/null +++ b/2014-Group5/Matlab/PolynomialFeatureMap.m @@ -0,0 +1,17 @@ +function Features = PolynomialFeatureMap(X,Y) + Features = zeros(size(X,1), 1+size(X,2)+size(Y,2)+size(X,2)*size(Y,2)); + for i=1:size(X,1); + Features(i,1) = 1; + Features(i,2:(size(X,2)+1)) = X(i,:); + Features(i,(size(X,2)+2):(size(X,2)+1+size(Y,2))) = Y(i,:); + + c = size(X,2)+1+size(Y,2)+1; + for j=1:size(X,2); + for k=1:size(Y,2); + Features(i,c) = X(i,j)*Y(i,k); + c = c + 1; + end; + end; + end; +end + diff --git a/2014-Group5/Matlab/PolynomialFeaturesLassoModel.m b/2014-Group5/Matlab/PolynomialFeaturesLassoModel.m new file mode 100644 index 0000000..7752955 --- /dev/null +++ b/2014-Group5/Matlab/PolynomialFeaturesLassoModel.m @@ -0,0 +1,220 @@ +% We build the linear lasso regression model based on the features extracted from PPCA and polynomial feature map. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. PolynomialFeaturesLassoModel +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +stockname = 'IBM'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +TwitterFeatureNames = {'Friends x Tweets,Stock Symbol, All Sentiments' ... + 'Friends x Tweets,Company Name,Negative', ... + 'Friends x Tweets,Company Name,NeTotalTweetsCountutral', ... + 'Friends x Tweets,Company Name,Positive'}; + +ExtractedTwitterFeatures = [TwitterFeatures(:, 13) + TwitterFeatures(:, 19) + TwitterFeatures(:, 14) + TwitterFeatures(:, 20) ... + + TwitterFeatures(:, 15) + TwitterFeatures(:, 21), TwitterFeatures(:, 16) + TwitterFeatures(:, 22), TwitterFeatures(:, 17) + TwitterFeatures(:, 23) ... + , TwitterFeatures(:, 18) + TwitterFeatures(:, 24)]; + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Normalize all features to have zero mean and standard deviation one +ExtractedTwitterFeatures = zscore(ExtractedTwitterFeatures); +StockFeatures = zscore(StockFeatures); + + +PolynomialFeatures = PolynomialFeatureMap(ExtractedTwitterFeatures, StockFeatures); +[COEFF,SCORE] = ppca(zscore(PolynomialFeatures(1:TrainingDataSize,:)), 4); +PolynomialFeatures = zscore(PolynomialFeatures*COEFF); +%PolynomialFeatures = zscore(PolynomialFeatures); +%PolynomialFeatures = zscore(TwitterFeatures); + + +% Build linear model separately for each dependent variable +PredictedTables = zeros(TestDataSize, 4); +TwitterDaysUsed = zeros(TestDataSize, 4); +for j = 1:4 + % For each test sample, we use leave-one-out cross-validation to pick + % the optimal m (0-3) and n (0-3) and use the best model to predict the + % next future value. + for i = (TotalDataSize-TestDataSize+1):TotalDataSize + i + % We use leave-one-out cross-validation on the previous 10 samples to find the optimal m and n + CrossValidationResults=zeros(3,4); + + OptimalSquaredError = 10000000; + Optimalm = 1; + Optimaln = 0; + % We vary the number of previous stock feature inputs + for m=1:3 + %m + % We vary the number of previous Twitter feature inputs + for n=0:1 + %n + for l=(i-10):i + TrainingInput = []; + ValidationInput = []; + + for k=1:m + TrainingInput = [TrainingInput, StockFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, StockFeatures((l-k),:)]; + end; + + for k=1:n + TrainingInput = [TrainingInput, PolynomialFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, PolynomialFeatures((l-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(l-1),j); + ValidationOutput = StockFeatures(l,j); + + CrossValidationPrediction = LinearLasso(TrainingInput, ValidationInput, TrainingOutput); + + %CrossValidationResults(l-(i-10)+1,m,n+1) = (ValidationOutput-CrossValidationPrediction).^2; + CrossValidationResults(m,n+1) = CrossValidationResults(m,n+1) + (ValidationOutput-CrossValidationPrediction).^2; + end; + + + if (OptimalSquaredError > CrossValidationResults(m,n+1)) + OptimalSquaredError = CrossValidationResults(m,n+1); + Optimalm = m; + Optimaln = n; + end; + end; + end; + + % We use the optimal n and m found from cross-validation to build a + % new linear regression model based on all the previous data + % samples, and use it to predict the next sample. + TrainingInput = []; + TestInput = []; + + for k=1:Optimalm + TrainingInput = [TrainingInput, StockFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, StockFeatures((l-k),:)]; + end; + + for k=1:Optimaln + TrainingInput = [TrainingInput, PolynomialFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, PolynomialFeatures((i-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(i-1),j); + + PredictedTables(i-(TotalDataSize-TestDataSize),j) = LinearLasso(TrainingInput, TestInput, TrainingOutput); + + TwitterDaysUsed(i-(TotalDataSize-TestDataSize),j) = Optimaln; + end; +end; + +% Save results +save(strcat('Lasso_PolynomialFeatures_PredictionTable_', stockname, ''), 'PredictedTables', 'TwitterDaysUsed'); + +% Plot the results. This is good for the report and will make it easy to +% interpret the model performance. + +h=figure; +%suptitle('Predictions for General Electric', 'FontType', 'Times'); +subplot(2,2,1); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,1), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,1), 'LineWidth', 2); +ylabel('Volume (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,2); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,2), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,2), 'LineWidth', 2); +ylabel('Closing Price (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,3); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,3), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,3), 'LineWidth', 2); +ylabel('Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,4); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,4), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,4), 'LineWidth', 2); +ylabel('Abs Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Predictions for ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('Predictions_', stockname, '')); +saveas(h,strcat('Predictions_', stockname, ''),'png'); + +% Calculate mean absolute error, squared error and accuracy (positive versus negative) +fprintf('Trading Volume (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1))), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,1) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + + +fprintf('Closing Price (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2))), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,2) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3))), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,3) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Abs Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4))), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,4) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); diff --git a/2014-Group5/Matlab/PolynomialFeaturesLinearModel.m b/2014-Group5/Matlab/PolynomialFeaturesLinearModel.m new file mode 100644 index 0000000..4f8d5d9 --- /dev/null +++ b/2014-Group5/Matlab/PolynomialFeaturesLinearModel.m @@ -0,0 +1,218 @@ +% We build the least-squared linear regression model based on the features extracted from PPCA and polynomial feature map. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. PolynomialFeaturesLassoModel +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +stockname = 'Intel'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +TwitterFeatureNames = {'Friends x Tweets,Stock Symbol, All Sentiments' ... + 'Friends x Tweets,Company Name,Negative', ... + 'Friends x Tweets,Company Name,NeTotalTweetsCountutral', ... + 'Friends x Tweets,Company Name,Positive'}; + +ExtractedTwitterFeatures = [TwitterFeatures(:, 13) + TwitterFeatures(:, 19) + TwitterFeatures(:, 14) + TwitterFeatures(:, 20) ... + + TwitterFeatures(:, 15) + TwitterFeatures(:, 21), TwitterFeatures(:, 16) + TwitterFeatures(:, 22), TwitterFeatures(:, 17) + TwitterFeatures(:, 23) ... + , TwitterFeatures(:, 18) + TwitterFeatures(:, 24)]; + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Normalize all features to have zero mean and standard deviation one +ExtractedTwitterFeatures = zscore(ExtractedTwitterFeatures); +StockFeatures = zscore(StockFeatures); + + +PolynomialFeatures = PolynomialFeatureMap(ExtractedTwitterFeatures, StockFeatures); +[COEFF,SCORE] = ppca(zscore(PolynomialFeatures(1:TrainingDataSize,:)), 4); +PolynomialFeatures = zscore(PolynomialFeatures*COEFF); + + +% Build linear model separately for each dependent variable +PredictedTables = zeros(TestDataSize, 4); +TwitterDaysUsed = zeros(TestDataSize, 4); +for j = 1:4 + % For each test sample, we use leave-one-out cross-validation to pick + % the optimal m (0-3) and n (0-3) and use the best model to predict the + % next future value. + for i = (TotalDataSize-TestDataSize+1):TotalDataSize + i + % We use leave-one-out cross-validation on the previous 10 samples to find the optimal m and n + CrossValidationResults=zeros(3,4); + + OptimalSquaredError = 10000000; + Optimalm = 1; + Optimaln = 0; + % We vary the number of previous stock feature inputs + for m=1:3 + %m + % We vary the number of previous Twitter feature inputs + for n=0:1 + %n + for l=(i-10):i + TrainingInput = []; + ValidationInput = []; + + for k=1:m + TrainingInput = [TrainingInput, StockFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, StockFeatures((l-k),:)]; + end; + + for k=1:n + TrainingInput = [TrainingInput, PolynomialFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, PolynomialFeatures((l-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(l-1),j); + ValidationOutput = StockFeatures(l,j); + + CrossValidationPrediction = LinearRR(TrainingInput, ValidationInput, TrainingOutput,1^(-10)); + + %CrossValidationResults(l-(i-10)+1,m,n+1) = (ValidationOutput-CrossValidationPrediction).^2; + CrossValidationResults(m,n+1) = CrossValidationResults(m,n+1) + (ValidationOutput-CrossValidationPrediction).^2; + end; + + + if (OptimalSquaredError > CrossValidationResults(m,n+1)) + OptimalSquaredError = CrossValidationResults(m,n+1); + Optimalm = m; + Optimaln = n; + end; + end; + end; + + % We use the optimal n and m found from cross-validation to build a + % new linear regression model based on all the previous data + % samples, and use it to predict the next sample. + TrainingInput = []; + TestInput = []; + + for k=1:Optimalm + TrainingInput = [TrainingInput, StockFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, StockFeatures((l-k),:)]; + end; + + for k=1:Optimaln + TrainingInput = [TrainingInput, PolynomialFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, PolynomialFeatures((i-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(i-1),j); + + PredictedTables(i-(TotalDataSize-TestDataSize),j) = LinearRR(TrainingInput, TestInput, TrainingOutput,1^(-10)); + + TwitterDaysUsed(i-(TotalDataSize-TestDataSize),j) = Optimaln; + end; +end; + +% Save results +save(strcat('Linear_PolynomialFeatures_PredictionTable_', stockname, ''), 'PredictedTables', 'TwitterDaysUsed'); + +% Plot the results. This is good for the report and will make it easy to +% interpret the model performance. + +h=figure; +%suptitle('Predictions for General Electric', 'FontType', 'Times'); +subplot(2,2,1); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,1), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,1), 'LineWidth', 2); +ylabel('Volume (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,2); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,2), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,2), 'LineWidth', 2); +ylabel('Closing Price (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,3); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,3), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,3), 'LineWidth', 2); +ylabel('Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +subplot(2,2,4); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,4), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,4), 'LineWidth', 2); +ylabel('Abs Price Change (Normalized)'); +xlabel('Test Day'); +legend('Benchmark Model', 'Ground Truth'); + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Predictions for ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('Predictions_', stockname, '')); +saveas(h,strcat('Predictions_', stockname, ''),'png'); + +% Calculate mean absolute error, squared error and accuracy (positive versus negative) +fprintf('Trading Volume (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1))), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,1) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + + +fprintf('Closing Price (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2))), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,2) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3))), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,3) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Abs Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4))), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,4) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); diff --git a/2014-Group5/Matlab/ProcessTwitterCSVFile.m b/2014-Group5/Matlab/ProcessTwitterCSVFile.m new file mode 100644 index 0000000..e86c909 --- /dev/null +++ b/2014-Group5/Matlab/ProcessTwitterCSVFile.m @@ -0,0 +1,110 @@ +% We load in file CSV file. It has columns Company Index, Month, Day, +% Features. + +A = importdata('complete.csv',','); + +Features = A(:,4:end); +CompanyDateIndex = [A(:,3), A(:,2), A(:,1)]; + +StillNeedToSum = true; + +% We sum up all entries for same company and dates +while (StillNeedToSum==true) + StillNeedToSum = false; + for i=1:size(CompanyDateIndex,1); + IndicesSameCompanySameDate = find(CompanyDateIndex(:,1) == CompanyDateIndex(i,1) & CompanyDateIndex(:,2) == CompanyDateIndex(i,2) & CompanyDateIndex(:,3) == CompanyDateIndex(i,3)); + IndicesSameCompanySameDate = sort(IndicesSameCompanySameDate); + + if (length(IndicesSameCompanySameDate) > 1) + % Sum up + Features(IndicesSameCompanySameDate(1),:) = sum(Features(IndicesSameCompanySameDate,:)); + + % Remove all other entries + Features(IndicesSameCompanySameDate(2:end),:) = []; + CompanyDateIndex(IndicesSameCompanySameDate(2:end),:) = []; + + StillNeedToSum = true; + + break; + end; + end; + +end; + +% We change our dates into indices. We count January 13th as the first day. +CompanyDay = zeros(size(CompanyDateIndex,1), 2); +for i=1:size(CompanyDateIndex,1); + CompanyDay(i,1) = CompanyDateIndex(i,1); + + numdays = datenum(strcat(num2str(CompanyDateIndex(i,2)), '/', num2str(CompanyDateIndex(i,3)), '/2014')) - datenum('01/12/2014'); + CompanyDay(i,2) = numdays; +end; + +% Add zeros to days with missing entries +for i=1:50; + for j=1:3; + if length(find(CompanyDay(:,1) == j & CompanyDay(:,2) == i)) == 0 + CompanyDay = [CompanyDay; [j,i]]; + Features = [Features; zeros(1,size(Features,2))]; + end; + end; +end; + + +% Finally we sort them in chronological order +[~, SortedIndices] = sort(CompanyDay(:,2)); +CompanyDay(:,1:2) = CompanyDay(SortedIndices,1:2); +Features(:,1:end) = Features(SortedIndices,1:end); + +TwitterFeaturesIBM = Features(find(CompanyDay(:,1) == 1),:); +TwitterFeaturesIntel = Features(find(CompanyDay(:,1) == 2),:); +TwitterFeaturesGE = Features(find(CompanyDay(:,1) == 3),:); + +save('TwitterFeaturesIBM.mat', 'TwitterFeaturesIBM'); +save('TwitterFeaturesIntel.mat', 'TwitterFeaturesIntel'); +save('TwitterFeaturesGE.mat', 'TwitterFeaturesGE'); + + +%CompanyOneIndices = find(CompanyDay(:,1) == 1); +%CompanyTwoIndices = find(CompanyDay(:,1) == 2); +%CompanyThreeIndices = find(CompanyDay(:,1) == 3); +%plot(CompanyDay(CompanyOneIndices,2), Features(CompanyOneIndices, 1), CompanyDay(CompanyOneIndices,2), Features(CompanyOneIndices, 2), CompanyDay(CompanyOneIndices,2), Features(CompanyOneIndices, 3)); +%legend('Negative', 'Neutral', 'Positive'); + +% To count the total number of tweets we need to sum features 1-3 and 7-9 +%TotalTweetsCount = zeros(49,3); +%TotalTweetsCount(:,1) = sum(Features(CompanyOneIndices, [1,2,3,7,8,9])'); +%TotalTweetsCount(:,2) = sum(Features(CompanyTwoIndices, [1,2,3,7,8,9])'); +%TotalTweetsCount(:,3) = sum(Features(CompanyThreeIndices, [1,2,3,7,8,9])'); + +%plot(1:size(TotalTweetsCount,1), TotalTweetsCount(:,1), 1:size(TotalTweetsCount,1), TotalTweetsCount(:,2), 1:size(TotalTweetsCount,1), TotalTweetsCount(:,3)); +%legend('IBM', 'Intel', 'General Electric'); +%xlabel('Day'); +%ylabel('Total Tweets'); + +% + +% Let's load the stock data. +% Stock features: (volume traded, closing price, daily change price, abs +% daily change price) +%[IBM, Intel, GE] = load_3_stocks; + +%TotalTweetsCount = [TotalTweetsCount(1:11,:); zeros(1,3); TotalTweetsCount(12:end,:)]; + +%scatter(zscore(TotalTweetsCount(:,2)), zscore(Intel(:,1))); +%scatter(zscore(TotalTweetsCount(1:49,2)), zscore(Intel(2:end,1))); +%xlabel('Intel Tweets Count'); +%ylabel('Intel Change in closing price'); + +%hold on; +%bar(1:50,zscore(TotalTweetsCount(:,2))); +%plot(1:50, zscore(Intel(:,1)), 'r'); +%legend('Intel Tweets Count', 'Intel Volume'); +%xlabel('Day'); + + +%hold off; + +% Calculate correlation coefficients +%corrcoef(zscore(TotalTweetsCount(:,1)), zscore(Intel(:,1))) + diff --git a/2014-Group5/Matlab/RandomSimulations.m b/2014-Group5/Matlab/RandomSimulations.m new file mode 100644 index 0000000..3627802 --- /dev/null +++ b/2014-Group5/Matlab/RandomSimulations.m @@ -0,0 +1,11 @@ + +t = randn(10000,1); +p = randn(10000,1); + + +fprintf('Random Model \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f %8.3f \n', mean(abs(t - p)), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f %8.3f \n', mean((t - p).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); +fprintf('Accuracy (Pos vs. Neg) %8.3f n/a \n \n', length(find(t.*p >= 0))/10000); + diff --git a/2014-Group5/Matlab/Readme.txt b/2014-Group5/Matlab/Readme.txt new file mode 100644 index 0000000..e9fb0bf --- /dev/null +++ b/2014-Group5/Matlab/Readme.txt @@ -0,0 +1,60 @@ +The src document applies the source codes for the coursework such as analysis the raw twitter data and the sentiment analysis platforms, Stanford and Lingpipe. + + +- BasicLassoModel.m +Code for the linear regression model based on tweet counts only and stock features with LASSO regularization. + +- BasicLinearModel.m +Code for the linear regression model based on tweet counts only and stock features with no regularization. + +- commands.txt +Misc commands used to edit raw Twitter data. + +- complete.csv +72 extracted Twitter features. + +- ConstantModel.m +Code for the Constant Model. + +- ExtrapolateStockData.m +Uses an AR(2) model to extrapolate the stock data over weekends and holidays. This gives us a total of 50 training days. + +- FeatureAnalysis.m +Performs the feature analysis described in the report. + +- LinearLasso.m +Performs a linear regression with LASSO regularization on arbitrary input data. + +- LinearLasso.m +Performs a linear regression with LASSO regularization on arbitrary input data. + +- LinearRR.m +Performs a linear ridge regression on arbitrary input data. We set the alpha parameter strictly positive but close to zero to achieve unique solution. + +- Load_3_stocks.m +Initial code not used anymore. + +- PolynomialFeatureMap.m +Polynomial feature function which transforms (X,Y) input into (X,Y, X_1 Y_1, X_2 Y_2 etc.) inputs. + +- PolynomialFeaturesLassoModel.m +Code for the linear regression model based on PPCA features and stock features with LASSO regularization. + +- PolynomialFeaturesLinearModel.m +Code for the linear regression model based on PPCA features and stock features with no regularization. + +- ProcessTwitterCSVFile.m +Converts Twitter features into Matlab matrices for further processing. + +- RandomSimulations.m +Simulates the N(0,1) Random Model and gives its MAE, MSE and Accuracy + +- SelectedFeaturesPlot.m +Plots the four selected features as discussed in the report. + +- SentimentLassoModel.m +Code for the linear regression model based on four selected Twitter features and stock features with LASSO regularization. + +- SentimentLinearModel.m +Code for the linear regression model based on four selected Twitter features and stock features with no regularization. + diff --git a/2014-Group5/Matlab/SelectedFeaturesPlot.m b/2014-Group5/Matlab/SelectedFeaturesPlot.m new file mode 100644 index 0000000..5904508 --- /dev/null +++ b/2014-Group5/Matlab/SelectedFeaturesPlot.m @@ -0,0 +1,151 @@ +% We build the baselinle / benchmark linear model here. This is the same +% as the one proposed by Mao et al. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +modelname = 'Linear Sentiment Model'; +stockname = 'GE'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +%TwitterFeatureNames = {'Friends x Tweets,Stock Symbol, All Sentiments' ... +% 'Friends x Tweets,Company Name,Negative', ... +% 'Friends x Tweets,Company Name,Neutral', ... +% 'Friends x Tweets,Company Name,Positive'}; + +TwitterFeatureNames = {'Stock Symbol, All' ... + 'Company Name,Negative', ... + 'Company Name,Neutral', ... + 'Company Name,Positive'}; + + +ExtractedTwitterFeatures = [TwitterFeatures(:, 13) + TwitterFeatures(:, 19) + TwitterFeatures(:, 14) + TwitterFeatures(:, 20) ... + + TwitterFeatures(:, 15) + TwitterFeatures(:, 21), TwitterFeatures(:, 16) + TwitterFeatures(:, 22), TwitterFeatures(:, 17) + TwitterFeatures(:, 23) ... + , TwitterFeatures(:, 18) + TwitterFeatures(:, 24)]; + + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Normalize all features to have zero mean and standard deviation one +ExtractedTwitterFeatures = zscore(ExtractedTwitterFeatures); +StockFeatures = zscore(StockFeatures); + + +TrainingDataIndices = 2:(TotalDataSize-TestDataSize); +scatter3(ExtractedTwitterFeatures(TrainingDataIndices-1, 1), StockFeatures(TrainingDataIndices-1, 3), StockFeatures(TrainingDataIndices, 3)); + +StockFeatureNames = {'Trading Volume', 'Closing Price', 'Price Change', 'Abs Price Change'}; + + +h=figure; +c = 1; +for i=1:4 + for j=1:4 + subplot(4,4,c); + scatter(ExtractedTwitterFeatures(TrainingDataIndices-1, i), StockFeatures(TrainingDataIndices, j)); + ylabel(StockFeatureNames{j}); + xlabel(TwitterFeatureNames{i}); + c = c+1; + end; +end; + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Twitter Features vs. Stock Features: ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('SelectedFeatures_', stockname, '')); +saveas(h,strcat('SelectedFeatures_', stockname, ''),'png'); + +h=figure; +scatter3(ExtractedTwitterFeatures(TrainingDataIndices-1, 1), ExtractedTwitterFeatures(TrainingDataIndices-1, 1), StockFeatures(TrainingDataIndices, 1)); +set(gca, 'FontSize', 13) +title(horzcat('Twitter Features vs. Stock Features: ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); +xlabel(horzcat(TwitterFeatureNames{1}, ', time t-1')); +ylabel(horzcat(TwitterFeatureNames{2}, ' , time t-1')); +zlabel(horzcat(StockFeatureNames{1}, ', time t')); + + +h=figure; +scatter3(ExtractedTwitterFeatures(TrainingDataIndices-1, 1), StockFeatures(TrainingDataIndices-1, 1), StockFeatures(TrainingDataIndices, 1)); +set(gca, 'FontSize', 13) +title(horzcat('Twitter Features vs. Stock Features: ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); +xlabel(horzcat(TwitterFeatureNames{1}, ', time t-1')); +ylabel(horzcat(StockFeatureNames{1}, ' , time t-1')); +zlabel(horzcat(StockFeatureNames{1}, ', time t')); + +h=figure;StockFeatures +scatter3(ExtractedTwitterFeatures(TrainingDataIndices-1, 1), StockFeatures(TrainingDataIndices-1, 1), StockFeatures(TrainingDataIndices, 3)); +set(gca, 'FontSize', 13) +title(horzcat('Twitter Features vs. Stock Features: ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); +xlabel(horzcat(TwitterFeatureNames{1}, ', time t-1')); +ylabel(horzcat(StockFeatureNames{1}, ' , time t-1')); +zlabel(horzcat(StockFeatureNames{3}, ', time t')); + +h=figure; +scatter3(ExtractedTwitterFeatures(TrainingDataIndices-1, 3), StockFeatures(TrainingDataIndices-1, 1), StockFeatures(TrainingDataIndices, 3)); +set(gca, 'FontSize', 13) +title(horzcat('Twitter Features vs. Stock Features: ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); +xlabel(horzcat(TwitterFeatureNames{3}, ', time t-1')); +ylabel(horzcat(StockFeatureNames{1}, ' , time t-1')); +zlabel(horzcat(StockFeatureNames{3}, ', time t'));2:(TotalDataSize-TestDataSize); + + + +PolynomialFeatures = PolynomialFeatureMap(ExtractedTwitterFeatures(1:TrainingDataSize, :), StockFeatures(1:TrainingDataSize, :)); +%[COEFF,SCORE] = princomp(zscore(PolynomialFeatures)); +[COEFF,SCORE] = ppca(zscore(PolynomialFeatures), 4); + +h=figure; +c = 1; +for i=1:4 + for j=1:4 + subplot(4,4,c); + scatter(SCORE(TrainingDataIndices-1, i), StockFeatures(TrainingDataIndices, j)); + ylabel(StockFeatureNames{j}) + xlabel(horzcat('Component ', num2str(i))); + c = c+1; + end; +end; + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Polynomial Features (PPCA) vs. Stock Features: ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('PolynomialFeatures_', stockname, '')); +saveas(h,strcat('PolynomialFeatures_', stockname, ''),'png'); \ No newline at end of file diff --git a/2014-Group5/Matlab/SentimentLassoModel.m b/2014-Group5/Matlab/SentimentLassoModel.m new file mode 100644 index 0000000..4a2b723 --- /dev/null +++ b/2014-Group5/Matlab/SentimentLassoModel.m @@ -0,0 +1,209 @@ +% We build the linear lasso regression model based on tweet sentiments. +% as the one proposed by Mao et al. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +modelname = 'Linear Sentiment Model'; +stockname = 'GE'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +TwitterFeatureNames = {'Friends x Tweets,Stock Symbol, All Sentiments' ... + 'Friends x Tweets,Company Name,Negative', ... + 'Friends x Tweets,Company Name,Neutral', ... + 'Friends x Tweets,Company Name,Positive'}; + +ExtractedTwitterFeatures = [TwitterFeatures(:, 13) + TwitterFeatures(:, 19) + TwitterFeatures(:, 14) + TwitterFeatures(:, 20) ... + + TwitterFeatures(:, 15) + TwitterFeatures(:, 21), TwitterFeatures(:, 16) + TwitterFeatures(:, 22), TwitterFeatures(:, 17) + TwitterFeatures(:, 23) ... + , TwitterFeatures(:, 18) + TwitterFeatures(:, 24)]; + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Normalize all features to have zero mean and standard deviation one +ExtractedTwitterFeatures = zscore(ExtractedTwitterFeatures); +StockFeatures = zscore(StockFeatures); + + +% Build linear model separately for each dependent variable +PredictedTables = zeros(TestDataSize, 4); +TwitterDaysUsed = zeros(TestDataSize, 4); +for j = 1:4 + % For each test sample, we use leave-one-out cross-validation to pick + % the optimal m (0-3) and n (0-3) and use the best model to predict the + % next future value. + for i = (TotalDataSize-TestDataSize+1):TotalDataSize + i + % We use leave-one-out cross-validation on the previous 10 samples to find the optimal m and n + CrossValidationResults=zeros(3,4); + + OptimalSquaredError = 10000000; + Optimalm = 1; + Optimaln = 0; + % We vary the number of previous stock feature inputs + for m=1:3 + % We vary the number of previous Twitter feature inputs + for n=0:3 + for l=(i-10):i + TrainingInput = []; + ValidationInput = []; + + for k=1:m + TrainingInput = [TrainingInput, StockFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, StockFeatures((l-k),:)]; + end; + + for k=1:n + TrainingInput = [TrainingInput, ExtractedTwitterFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, ExtractedTwitterFeatures((l-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(l-1),j); + ValidationOutput = StockFeatures(l,j); + + CrossValidationPrediction = LinearLasso(TrainingInput, ValidationInput, TrainingOutput); + + %CrossValidationResults(l-(i-10)+1,m,n+1) = (ValidationOutput-CrossValidationPrediction).^2; + CrossValidationResults(m,n+1) = CrossValidationResults(m,n+1) + (ValidationOutput-CrossValidationPrediction).^2; + end; + + if (OptimalSquaredError > CrossValidationResults(m,n+1)) + OptimalSquaredError = CrossValidationResults(m,n+1); + Optimalm = m; + Optimaln = n; + end; + end; + end; + + % We use the optimal n and m found from cross-validation to build a + % new linear regression model based on all the previous data + % samples, and use it to predict the next sample. + TrainingInput = []; + TestInput = []; + + for k=1:Optimalm + TrainingInput = [TrainingInput, StockFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, StockFeatures((l-k),:)]; + end; + + for k=1:Optimaln + TrainingInput = [TrainingInput, ExtractedTwitterFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, ExtractedTwitterFeatures((i-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(i-1),j); + + PredictedTables(i-(TotalDataSize-TestDataSize),j) = LinearLasso(TrainingInput, TestInput, TrainingOutput); + + TwitterDaysUsed(i-(TotalDataSize-TestDataSize),j) = Optimaln; + end; +end; + +% Plot the results. This is good for the report and will make it easy to +% interpret the model performance. + +h=figure; +%suptitle('Predictions for General Electric', 'FontType', 'Times'); +subplot(2,2,1); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,1), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,1), 'LineWidth', 2); +ylabel('Volume (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +subplot(2,2,2); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,2), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,2), 'LineWidth', 2); +ylabel('Closing Price (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +subplot(2,2,3); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,3), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,3), 'LineWidth', 2); +ylabel('Price Change (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +subplot(2,2,4); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,4), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,4), 'LineWidth', 2); +ylabel('Abs Price Change (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Predictions for ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('Predictions_', modelname, '_', stockname, '')); +saveas(h,strcat('Predictions_', modelname, '_', stockname, ''),'png'); + +% Calculate mean absolute error, squared error and accuracy (positive versus negative) +fprintf('Trading Volume (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1))), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,1) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + + +fprintf('Closing Price (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2))), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,2) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3))), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,3) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Abs Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4))), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,4) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); diff --git a/2014-Group5/Matlab/SentimentLinearModel.m b/2014-Group5/Matlab/SentimentLinearModel.m new file mode 100644 index 0000000..107689f --- /dev/null +++ b/2014-Group5/Matlab/SentimentLinearModel.m @@ -0,0 +1,208 @@ +% We build the least-squared linear regression model based on tweet sentiments. + +% We have 72 features for each stock. Imagine them as a tree: + +% - Weighting Factor (x1, x number of friends, x log2(number of friends), x1, x number of followers, x log2(number of followers) +% - - Contains url (no url in text, url in text) +% - - - Tweet type (mentions stock symbol, mentions company name) +% - - - - Sentiment (negative, neutral, positive) + +% We create an 72x1 array which describes each feature index +WeightDescriptions = {'1 x ', 'Friends x ', 'log2(Friends + 1) x ', '1 x ', 'Followers x ', 'log2(Followers + 1) x '}; +ContainsUrl = {'No URL', 'URL'}; +TweetType = {'Stock Symbol', 'Company Name'}; +Sentiment = {'Negative', 'Neutral', 'Positive'}; + +FeatureNameDescription = cell(72,1); +c = 1; +for i=1:6 % Weight factor + for j=1:2 % Contains url + for k=1:2 % Tweet type + for l=1:3 % sentiment + FeatureNameDescription{c} = strcat(WeightDescriptions{i}, ' Tweets, ', ContainsUrl{j}, ', ', TweetType{k}, ', ', Sentiment{l}); + c = c+1; + end; + end; + end; +end; + +% The tree has just been folded out to one long vector from the top. +% Starting with (Weight x1, no url, mentions stock symbol, negative), +% (Weight x1, no url, mentions stock symbol, neutral), (Weight x1, no url, mentions stock symbol, positive) etc. + +% Set training and test data size ssets +TotalDataSize = 50; +TrainingDataSize = 35; +TestDataSize = 15; + +modelname = 'Linear Sentiment Model'; +stockname = 'GE'; % Change this to 'IBM', 'Intel' or 'GE' + +TwitterFeatures = load(strcat('TwitterFeatures/TwitterFeatures', stockname, '.mat')); +TwitterFeatures = eval(strcat('TwitterFeatures.TwitterFeatures', stockname)); + +TwitterFeatureNames = {'Friends x Tweets,Stock Symbol, All Sentiments' ... + 'Friends x Tweets,Company Name,Negative', ... + 'Friends x Tweets,Company Name,Neutral', ... + 'Friends x Tweets,Company Name,Positive'}; + +ExtractedTwitterFeatures = [TwitterFeatures(:, 13) + TwitterFeatures(:, 19) + TwitterFeatures(:, 14) + TwitterFeatures(:, 20) ... + + TwitterFeatures(:, 15) + TwitterFeatures(:, 21), TwitterFeatures(:, 16) + TwitterFeatures(:, 22), TwitterFeatures(:, 17) + TwitterFeatures(:, 23) ... + , TwitterFeatures(:, 18) + TwitterFeatures(:, 24)]; + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +StockFeatures = load(strcat('InterpolatedStockFeatures/', stockname, 'Features.mat')); +StockFeatures = StockFeatures.StockFeatures; + +% Normalize all features to have zero mean and standard deviation one +ExtractedTwitterFeatures = zscore(ExtractedTwitterFeatures); +StockFeatures = zscore(StockFeatures); + + +% Build linear model separately for each dependent variable +PredictedTables = zeros(TestDataSize, 4); +TwitterDaysUsed = zeros(TestDataSize, 4); +for j = 1:4 + % For each test sample, we use leave-one-out cross-validation to pick + % the optimal m (0-3) and n (0-3) and use the best model to predict the + % next future value. + for i = (TotalDataSize-TestDataSize+1):TotalDataSize + i + % We use leave-one-out cross-validation on the previous 10 samples to find the optimal m and n + CrossValidationResults=zeros(3,4); + + OptimalSquaredError = 10000000; + Optimalm = 1; + Optimaln = 0; + % We vary the number of previous stock feature inputs + for m=1:3 + % We vary the number of previous Twitter feature inputs + for n=0:3 + for l=(i-10):i + TrainingInput = []; + ValidationInput = []; + + for k=1:m + TrainingInput = [TrainingInput, StockFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, StockFeatures((l-k),:)]; + end; + + for k=1:n + TrainingInput = [TrainingInput, ExtractedTwitterFeatures((4-k):(l-k-1),:)]; + ValidationInput = [ValidationInput, ExtractedTwitterFeatures((l-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(l-1),j); + ValidationOutput = StockFeatures(l,j); + + CrossValidationPrediction = LinearRR(TrainingInput, ValidationInput, TrainingOutput,1^(-10)); + + %CrossValidationResults(l-(i-10)+1,m,n+1) = (ValidationOutput-CrossValidationPrediction).^2; + CrossValidationResults(m,n+1) = CrossValidationResults(m,n+1) + (ValidationOutput-CrossValidationPrediction).^2; + end; + + if (OptimalSquaredError > CrossValidationResults(m,n+1)) + OptimalSquaredError = CrossValidationResults(m,n+1); + Optimalm = m; + Optimaln = n; + end; + end; + end; + + % We use the optimal n and m found from cross-validation to build a + % new linear regression model based on all the previous data + % samples, and use it to predict the next sample. + TrainingInput = []; + TestInput = []; + + for k=1:Optimalm + TrainingInput = [TrainingInput, StockFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, StockFeatures((l-k),:)]; + end; + + for k=1:Optimaln + TrainingInput = [TrainingInput, ExtractedTwitterFeatures((4-k):(i-k-1),:)]; + TestInput = [TestInput, ExtractedTwitterFeatures((i-k),:)]; + end; + + TrainingOutput = StockFeatures(4:(i-1),j); + + PredictedTables(i-(TotalDataSize-TestDataSize),j) = LinearRR(TrainingInput, TestInput, TrainingOutput,1^(-10)); + + TwitterDaysUsed(i-(TotalDataSize-TestDataSize),j) = Optimaln; + end; +end; + +% Plot the results. This is good for the report and will make it easy to +% interpret the model performance. + +h=figure; +%suptitle('Predictions for General Electric', 'FontType', 'Times'); +subplot(2,2,1); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,1), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,1), 'LineWidth', 2); +ylabel('Volume (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +subplot(2,2,2); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,2), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,2), 'LineWidth', 2); +ylabel('Closing Price (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +subplot(2,2,3); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,3), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,3), 'LineWidth', 2); +ylabel('Price Change (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +subplot(2,2,4); +set(gca, 'FontSize', 13) +plot(1:TestDataSize, PredictedTables(:,4), 1:TestDataSize, StockFeatures(TrainingDataSize+1:end,4), 'LineWidth', 2); +ylabel('Abs Price Change (Normalized)'); +xlabel('Test Day'); +legend(modelname, 'Ground Truth'); + +ax=axes('Units','Normal','Position',[.075 .075 .85 .85],'Visible','off'); +set(get(ax,'Title'),'Visible','on') +title(horzcat('Predictions for ', stockname), 'FontSize', 16, 'FontWeight', 'Bold'); + +savefig(h, strcat('Predictions_', modelname, '_', stockname, '')); +saveas(h,strcat('Predictions_', modelname, '_', stockname, ''),'png'); + +% Calculate mean absolute error, squared error and accuracy (positive versus negative) +fprintf('Trading Volume (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1))), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2), std(abs(PredictedTables(:,1) - StockFeatures(TrainingDataSize+1:end,1)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,1) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + + +fprintf('Closing Price (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2))), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2), std(abs(PredictedTables(:,2) - StockFeatures(TrainingDataSize+1:end,2)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,2) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3))), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2), std(abs(PredictedTables(:,3) - StockFeatures(TrainingDataSize+1:end,3)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,3) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); + +fprintf('Abs Price Change (Normalized): \n'); +fprintf('Metric Mean Std \n'); +fprintf('Abs Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4))), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)))); +fprintf('Squared Error %8.3f +/- %8.3f \n', mean(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2), std(abs(PredictedTables(:,4) - StockFeatures(TrainingDataSize+1:end,4)).^2)); + +p = length(find(PredictedTables(:,4).*StockFeatures(TrainingDataSize+1:end,4) >= 0))/TestDataSize; +fprintf('Accuracy (Pos vs. Neg) %8.3f +/- %8.3f \n\n', p, sqrt((p*(1-p))/TestDataSize)); diff --git a/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesGE.mat b/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesGE.mat new file mode 100644 index 0000000..0b85ea1 Binary files /dev/null and b/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesGE.mat differ diff --git a/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesIBM.mat b/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesIBM.mat new file mode 100644 index 0000000..6ca84ff Binary files /dev/null and b/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesIBM.mat differ diff --git a/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesIntel.mat b/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesIntel.mat new file mode 100644 index 0000000..09a1bf4 Binary files /dev/null and b/2014-Group5/Matlab/TwitterFeatures/TwitterFeaturesIntel.mat differ diff --git a/2014-Group5/Matlab/commands.txt b/2014-Group5/Matlab/commands.txt new file mode 100644 index 0000000..897bb22 --- /dev/null +++ b/2014-Group5/Matlab/commands.txt @@ -0,0 +1 @@ +cat store* > complete diff --git a/2014-Group5/Matlab/complete.csv b/2014-Group5/Matlab/complete.csv new file mode 100644 index 0000000..2fd4a40 --- /dev/null +++ b/2014-Group5/Matlab/complete.csv @@ -0,0 +1,366 @@ +25,1, 2, 0, 0, 0, 5, 16, 2, 0, 0, 0, 4, 8, 8, 0, 0, 0, 38058, 21770, 530, 0, 0, 0, 4114, 5992, 6394, 0, 0, 0, 50, 148, 16, 0, 0, 0, 30, 52, 74, 0, 0, 0, 5, 16, 2, 0, 0, 0, 4, 8, 8, 0, 0, 0, 41314, 21530, 2216, 0, 0, 0, 1186, 1298, 6944, 0, 0, 0, 50, 140, 20, 0, 0, 0, 32, 40, 68 +27,1, 1, 0, 0, 0, 240, 1324, 387, 0, 3, 0, 60, 1524, 261, 0, 0, 0, 243924, 1361171, 535176, 0, 14, 0, 43962, 1813708, 246795, 0, 0, 0, 1689, 9517, 3135, 0, 5, 0, 345, 11407, 1947, 0, 0, 0, 240, 1324, 387, 0, 3, 0, 60, 1524, 261, 0, 0, 0, 375612, 2872780, 1172298, 0, 408, 0, 57603, 5302089, 581199, 0, 0, 0, 1686, 9930, 3219, 0, 20, 0, 396, 12034, 2085 +26,1, 2, 0, 2, 0, 602, 526, 698, 0, 29, 0, 1086, 1792, 538, 0, 254, 0, 330016, 179396, 345560, 0, 4627, 0, 1981562, 2813356, 805248, 0, 14, 0, 4338, 2370, 4202, 0, 57, 0, 9300, 12825, 4102, 0, 2, 0, 602, 526, 698, 0, 29, 0, 1086, 1792, 538, 0, 804, 0, 814800, 383620, 740200, 0, 65924, 0, 3765692, 4350558, 2513918, 0, 16, 0, 4284, 2718, 4486, 0, 202, 0, 5742, 13467, 4082 +23,1, 3, 0, 0, 0, 1, 3, 0, 0, 2, 0, 0, 10, 3, 0, 0, 0, 203, 535, 0, 0, 15, 0, 0, 615, 2078, 0, 0, 0, 7, 19, 0, 0, 6, 0, 0, 39, 24, 0, 0, 0, 1, 3, 0, 0, 2, 0, 0, 10, 3, 0, 0, 0, 216, 839, 0, 0, 50025, 0, 0, 4194, 5609, 0, 0, 0, 7, 21, 0, 0, 23, 0, 0, 67, 28 +27,1, 3, 0, 0, 0, 19, 8, 0, 0, 6, 0, 2, 52, 4, 0, 0, 0, 11907, 976, 0, 0, 1357, 0, 0, 10287, 5025, 0, 0, 0, 150, 45, 0, 0, 21, 0, 0, 169, 31, 0, 0, 0, 19, 8, 0, 0, 6, 0, 2, 52, 4, 0, 0, 0, 21340, 13032, 0, 0, 2415, 0, 0, 19452, 5539, 0, 0, 0, 143, 54, 0, 0, 43, 0, 0, 271, 39 +25,1, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 360, 91713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 170, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 3, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 141752, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 167, 0 +27,1, 2, 0, 1, 0, 144, 93, 426, 0, 3, 0, 110, 532, 158, 0, 127, 0, 91186, 34771, 47896, 0, 5, 0, 127810, 610795, 677270, 0, 7, 0, 1018, 625, 964, 0, 2, 0, 790, 3618, 1148, 0, 1, 0, 144, 93, 426, 0, 3, 0, 110, 532, 158, 0, 407, 0, 265754, 375035, 71054, 0, 1086, 0, 387074, 875973, 855720, 0, 8, 0, 1036, 674, 988, 0, 19, 0, 936, 3827, 1156 +23,1, 1, 0, 2, 0, 369, 436, 57, 0, 36, 0, 153, 1414, 387, 0, 384, 0, 198804, 243306, 21246, 0, 5598, 0, 267114, 1185204, 499155, 0, 14, 0, 2574, 3142, 315, 0, 168, 0, 1143, 9368, 3156, 0, 2, 0, 369, 436, 57, 0, 36, 0, 153, 1414, 387, 0, 655, 0, 7210020, 989438, 56976, 0, 182307, 0, 3253863, 7728865, 1315893, 0, 15, 0, 2676, 3222, 273, 0, 356, 0, 1422, 10748, 3294 +23,1, 2, 0, 0, 0, 52, 52, 54, 0, 0, 0, 44, 262, 53, 0, 0, 0, 32028, 134038, 44898, 0, 0, 0, 25598, 219768, 43318, 0, 0, 0, 290, 424, 368, 0, 0, 0, 264, 1884, 366, 0, 0, 0, 52, 52, 54, 0, 0, 0, 44, 262, 53, 0, 0, 0, 42702, 178958, 59250, 0, 0, 0, 28866, 3004120, 60449, 0, 0, 0, 296, 412, 366, 0, 0, 0, 282, 2044, 380 +26,1, 1, 0, 0, 0, 549, 1713, 400, 0, 6, 0, 219, 2151, 315, 0, 0, 0, 318396, 1294281, 317062, 0, 2180, 0, 276792, 2681524, 530235, 0, 0, 0, 2955, 10086, 2458, 0, 31, 0, 1329, 15488, 2421, 0, 0, 0, 549, 1713, 400, 0, 6, 0, 219, 2151, 315, 0, 0, 0, 1238781, 2036112, 480110, 0, 3890, 0, 906336, 4108456, 1156806, 0, 0, 0, 3330, 10548, 2602, 0, 46, 0, 1605, 16004, 2472 +26,1, 3, 0, 0, 0, 4, 31, 2, 0, 2, 0, 22, 174, 18, 0, 0, 0, 3419, 3745, 1361, 0, 0, 0, 2038, 128692, 2731, 0, 0, 0, 31, 171, 17, 0, 0, 0, 74, 838, 48, 0, 0, 0, 4, 31, 2, 0, 2, 0, 22, 174, 18, 0, 0, 0, 3441, 5417, 1248, 0, 142, 0, 9478, 272954, 6641, 0, 0, 0, 30, 190, 17, 0, 10, 0, 88, 994, 83 +25,1, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 342, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 976, 0, 0, 415, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8, 0 +27,1, 1, 0, 0, 0, 1026, 5938, 1188, 0, 17, 0, 282, 4267, 717, 0, 0, 0, 830034, 8192420, 1821318, 0, 6486, 0, 293517, 5103465, 1369536, 0, 0, 0, 7119, 46495, 9459, 0, 88, 0, 1560, 31340, 5271, 0, 0, 0, 1026, 5938, 1188, 0, 17, 0, 282, 4267, 717, 0, 0, 0, 1486899, 12711503, 2572638, 0, 18552, 0, 398439, 10581012, 2533623, 0, 0, 0, 7326, 46869, 9687, 0, 137, 0, 1761, 32863, 5853 +28,1, 1, 0, 0, 0, 213, 786, 195, 0, 12, 0, 132, 1246, 198, 0, 0, 0, 117252, 809226, 148167, 0, 2864, 0, 105357, 1511416, 460635, 0, 0, 0, 1206, 5346, 1353, 0, 44, 0, 837, 8967, 1431, 0, 0, 0, 213, 786, 195, 0, 12, 0, 132, 1246, 198, 0, 0, 0, 137322, 4745661, 369789, 0, 49556, 0, 377595, 3816409, 935208, 0, 0, 0, 1215, 5649, 1299, 0, 105, 0, 1020, 9441, 1608 +27,1, 3, 0, 0, 0, 96, 25, 4, 0, 16, 0, 7, 215, 11, 0, 0, 0, 121232, 18420, 1514, 0, 5509, 0, 2294, 181439, 6050, 0, 0, 0, 784, 180, 31, 0, 41, 0, 28, 1073, 62, 0, 0, 0, 96, 25, 4, 0, 16, 0, 7, 215, 11, 0, 0, 0, 385714, 71605, 10238, 0, 32891, 0, 8502, 349869, 5837, 0, 0, 0, 767, 190, 36, 0, 131, 0, 41, 1462, 59 +27,1, 2, 0, 0, 0, 706, 550, 504, 0, 15, 0, 462, 2014, 628, 0, 0, 0, 433508, 178856, 284062, 0, 2961, 0, 426730, 2558617, 579264, 0, 0, 0, 5176, 2690, 3282, 0, 35, 0, 3400, 14743, 4546, 0, 0, 0, 706, 550, 504, 0, 15, 0, 462, 2014, 628, 0, 0, 0, 1543338, 463028, 999702, 0, 7275, 0, 2019902, 9224419, 880102, 0, 0, 0, 5294, 2904, 3476, 0, 101, 0, 3642, 15781, 4650 +28,1, 2, 0, 2, 0, 154, 118, 134, 0, 0, 0, 100, 435, 190, 0, 366, 0, 111268, 28886, 51564, 0, 0, 0, 456144, 419762, 201704, 0, 14, 0, 1148, 484, 672, 0, 0, 0, 808, 3063, 1426, 0, 2, 0, 154, 118, 134, 0, 0, 0, 100, 435, 190, 0, 1257, 0, 138862, 31265, 157802, 0, 0, 0, 627210, 1004799, 482654, 0, 17, 0, 1138, 531, 684, 0, 0, 0, 812, 3180, 1524 +28,1, 3, 0, 1, 0, 2, 10, 1, 0, 8, 0, 6, 56, 3, 0, 174, 0, 1954, 5645, 2, 0, 2148, 0, 513, 18175, 3492, 0, 7, 0, 18, 73, 1, 0, 39, 0, 24, 260, 18, 0, 1, 0, 2, 10, 1, 0, 8, 0, 6, 56, 3, 0, 1129, 0, 1403, 4254, 2532, 0, 17961, 0, 8193, 29728, 3466, 0, 10, 0, 18, 73, 11, 0, 74, 0, 49, 348, 20 +29,1, 2, 0, 0, 0, 132, 258, 144, 0, 4, 0, 76, 555, 212, 0, 0, 0, 83228, 145636, 85276, 0, 5557, 0, 31610, 341824, 829338, 0, 0, 0, 908, 1980, 972, 0, 27, 0, 412, 3477, 1488, 0, 0, 0, 132, 258, 144, 0, 4, 0, 76, 555, 212, 0, 0, 0, 244562, 721582, 533994, 0, 7000, 0, 105708, 1258802, 1622792, 0, 0, 0, 934, 2152, 1054, 0, 34, 0, 498, 3950, 1736 +29,1, 1, 0, 0, 0, 291, 2106, 144, 1, 9, 0, 210, 1638, 306, 0, 0, 0, 507033, 2064840, 285117, 867, 3395, 0, 180803, 1915666, 234318, 0, 0, 0, 1956, 15744, 1035, 9, 37, 0, 1486, 11612, 2067, 0, 0, 0, 291, 2106, 144, 1, 9, 0, 210, 1638, 306, 0, 0, 0, 5806170, 3839934, 416424, 740, 15009, 0, 301215, 3144222, 842685, 0, 0, 0, 2154, 15600, 1041, 9, 72, 0, 1702, 12594, 2412 +28,1, 1, 0, 1, 0, 537, 1988, 460, 0, 47, 0, 504, 5035, 753, 0, 0, 0, 502866, 2022957, 439309, 0, 11528, 0, 1089843, 6717160, 994674, 0, 0, 0, 3375, 13173, 3054, 0, 160, 0, 3372, 37553, 5505, 0, 1, 0, 537, 1988, 460, 0, 47, 0, 504, 5035, 753, 0, 139, 0, 833685, 4519463, 836966, 0, 42008, 0, 5347962, 18036616, 2175333, 0, 7, 0, 3570, 13463, 3096, 0, 354, 0, 3732, 39480, 5991 +28,1, 2, 0, 0, 0, 678, 556, 610, 0, 16, 0, 400, 1800, 765, 0, 0, 0, 422908, 327352, 319630, 0, 4447, 0, 327502, 2038749, 824374, 0, 0, 0, 4768, 3288, 3640, 0, 49, 0, 2726, 13119, 5659, 0, 0, 0, 678, 556, 610, 0, 16, 0, 400, 1800, 765, 0, 0, 0, 772980, 718064, 1094310, 0, 14765, 0, 1300028, 8951199, 1721741, 0, 0, 0, 4948, 3468, 3842, 0, 125, 0, 2812, 13581, 5960 +28,1, 3, 0, 0, 0, 8, 43, 14, 0, 29, 0, 14, 183, 10, 0, 0, 0, 4846, 47542, 13496, 0, 5929, 0, 6035, 63088, 1848, 0, 0, 0, 55, 314, 101, 0, 105, 0, 82, 781, 43, 0, 0, 0, 8, 43, 14, 0, 29, 0, 14, 183, 10, 0, 0, 0, 1803, 173202, 32267, 0, 160732, 0, 14568, 132484, 1097, 0, 0, 0, 48, 345, 111, 0, 263, 0, 110, 964, 31 +29,1, 3, 0, 0, 0, 0, 13, 4, 0, 3, 0, 1, 47, 11, 0, 0, 0, 0, 11991, 348, 0, 288, 0, 8, 25675, 137, 0, 0, 0, 0, 104, 22, 0, 14, 0, 3, 224, 20, 0, 0, 0, 0, 13, 4, 0, 3, 0, 1, 47, 11, 0, 0, 0, 0, 48261, 459, 0, 2656, 0, 18615, 51534, 8948, 0, 0, 0, 0, 108, 14, 0, 27, 0, 14, 258, 56 +29,1, 2, 0, 0, 0, 1338, 997, 1214, 0, 7, 0, 1118, 3441, 1272, 0, 0, 0, 1039558, 711846, 449394, 0, 20, 0, 1679646, 5332216, 1521832, 0, 0, 0, 10472, 7093, 5308, 0, 8, 0, 8346, 26344, 9258, 0, 0, 0, 1338, 997, 1214, 0, 7, 0, 1118, 3441, 1272, 0, 0, 0, 4237824, 7002834, 1855206, 0, 1537, 0, 5370488, 23717551, 5699006, 0, 0, 0, 10658, 7126, 5914, 0, 39, 0, 8590, 27973, 10180 +30,1, 1, 0, 2, 0, 99, 391, 75, 0, 19, 0, 288, 837, 102, 0, 71, 0, 54831, 151651, 36573, 0, 5660, 0, 268071, 537595, 95595, 0, 6, 0, 693, 2436, 504, 0, 76, 0, 2046, 5214, 786, 0, 2, 0, 99, 391, 75, 0, 19, 0, 288, 837, 102, 0, 332, 0, 354555, 283645, 56424, 0, 67515, 0, 466869, 2160750, 316287, 0, 14, 0, 771, 2419, 495, 0, 148, 0, 2262, 5982, 843 +29,1, 1, 0, 1, 0, 579, 2748, 531, 0, 38, 0, 480, 4792, 900, 0, 1191, 0, 553437, 1667423, 816750, 0, 8673, 0, 470952, 5335617, 1177641, 0, 10, 0, 3684, 19348, 3840, 0, 151, 0, 3375, 34274, 6513, 0, 1, 0, 579, 2748, 531, 0, 38, 0, 480, 4792, 900, 0, 150, 0, 1016961, 4973253, 1446642, 0, 41219, 0, 936708, 14238721, 2215173, 0, 7, 0, 3828, 19336, 4038, 0, 301, 0, 3783, 37409, 6801 +30,1, 2, 0, 0, 0, 196, 183, 352, 0, 0, 0, 122, 354, 132, 0, 0, 0, 73386, 139323, 16230, 0, 0, 0, 160880, 594040, 177928, 0, 0, 0, 1386, 1501, 694, 0, 0, 0, 828, 2748, 982, 0, 0, 0, 196, 183, 352, 0, 0, 0, 122, 354, 132, 0, 0, 0, 820418, 747258, 29118, 0, 0, 0, 329436, 2214054, 277372, 0, 0, 0, 1444, 1379, 918, 0, 0, 0, 994, 2872, 1052 +30,1, 3, 0, 0, 0, 1, 4, 0, 0, 2, 0, 0, 15, 0, 0, 0, 0, 337, 3012, 0, 0, 1, 0, 0, 1782, 0, 0, 0, 0, 8, 27, 0, 0, 1, 0, 0, 43, 0, 0, 0, 0, 1, 4, 0, 0, 2, 0, 0, 15, 0, 0, 0, 0, 116, 7312, 0, 0, 1108, 0, 0, 15275, 0, 0, 0, 0, 6, 32, 0, 0, 16, 0, 0, 38, 0 +29,1, 3, 0, 0, 0, 10, 58, 30, 0, 26, 0, 9, 149, 15, 0, 0, 0, 24269, 102877, 38545, 0, 4539, 0, 2038, 119872, 228896, 0, 0, 0, 87, 458, 235, 0, 79, 0, 33, 778, 90, 0, 0, 0, 10, 58, 30, 0, 26, 0, 9, 149, 15, 0, 0, 0, 33557, 565597, 42134, 0, 23681, 0, 1698, 232598, 229993, 0, 0, 0, 87, 462, 204, 0, 199, 0, 46, 872, 120 +30,1, 1, 0, 0, 0, 658, 2235, 708, 0, 25, 0, 702, 6548, 924, 0, 0, 0, 302556, 1549047, 654525, 0, 4649, 0, 428502, 7550833, 948588, 0, 0, 0, 4017, 14631, 4962, 0, 76, 0, 4293, 49262, 6591, 0, 0, 0, 658, 2235, 708, 0, 25, 0, 702, 6548, 924, 0, 0, 0, 1974807, 3316812, 1176855, 0, 52757, 0, 1608540, 17406013, 1790679, 0, 0, 0, 4074, 14787, 4914, 0, 205, 0, 4878, 51317, 7065 +31,1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0 +31,1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0 +30,1, 2, 0, 0, 0, 906, 914, 582, 0, 12, 0, 1516, 4313, 1306, 0, 0, 0, 753868, 969528, 666238, 0, 10644, 0, 2269756, 5985745, 2421840, 0, 0, 0, 6728, 7290, 4148, 0, 56, 0, 11472, 33090, 10698, 0, 0, 0, 906, 914, 582, 0, 12, 0, 1516, 4313, 1306, 0, 0, 0, 1909332, 1956710, 2102534, 0, 14703, 0, 4401754, 18764329, 9726242, 0, 0, 0, 6936, 7022, 4120, 0, 102, 0, 11794, 34425, 10924 +31,1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2770, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0 +30,1, 3, 0, 0, 0, 4, 87, 14, 0, 23, 0, 9, 230, 41, 0, 0, 0, 866, 157123, 17629, 0, 6474, 0, 4755, 225617, 178669, 0, 0, 0, 28, 657, 128, 0, 75, 0, 51, 1392, 359, 0, 0, 0, 4, 87, 14, 0, 23, 0, 9, 230, 41, 0, 0, 0, 993, 225790, 26693, 0, 69696, 0, 16162, 472250, 210815, 0, 0, 0, 27, 697, 128, 0, 179, 0, 66, 1558, 359 +30,1, 1, 0, 0, 0, 6, 42, 12, 0, 0, 0, 15, 54, 1, 0, 0, 0, 3531, 10566, 3084, 0, 0, 0, 10617, 192939, 28, 0, 0, 0, 51, 210, 81, 0, 0, 0, 126, 477, 4, 0, 0, 0, 6, 42, 12, 0, 0, 0, 15, 54, 1, 0, 0, 0, 1548, 8430, 21474, 0, 0, 0, 718851, 242649, 11, 0, 0, 0, 42, 219, 96, 0, 0, 0, 132, 480, 3 +31,1, 3, 0, 1, 0, 4, 29, 3, 1, 20, 0, 15, 291, 15, 0, 74, 0, 1409, 50804, 1566, 478, 744, 0, 7765, 292183, 13787, 0, 6, 0, 31, 223, 24, 8, 40, 0, 97, 1935, 110, 0, 1, 0, 4, 29, 3, 1, 20, 0, 15, 291, 15, 0, 2066, 0, 430, 63360, 1158, 177, 15174, 0, 16787, 1227076, 20141, 0, 11, 0, 25, 246, 23, 7, 150, 0, 103, 2175, 115 +31,1, 2, 0, 0, 0, 462, 1128, 278, 0, 4, 0, 226, 1291, 354, 0, 0, 0, 274848, 680862, 250086, 0, 6, 0, 236472, 1460770, 297232, 0, 0, 0, 3458, 8850, 2008, 0, 2, 0, 1646, 9274, 2582, 0, 0, 0, 462, 1128, 278, 0, 4, 0, 226, 1291, 354, 0, 0, 0, 993854, 1950678, 1199296, 0, 19025, 0, 499476, 3058344, 826700, 0, 0, 0, 3512, 8138, 2080, 0, 35, 0, 1764, 10195, 2872 +30,1, 2, 0, 0, 0, 6, 6, 5, 0, 0, 0, 14, 30, 4, 0, 0, 0, 2494, 17580, 846, 0, 0, 0, 5426, 95492, 13182, 0, 0, 0, 44, 64, 34, 0, 0, 0, 92, 262, 42, 0, 0, 0, 6, 6, 5, 0, 0, 0, 14, 30, 4, 0, 0, 0, 3150, 14142, 34106, 0, 0, 0, 1874, 97314, 15858, 0, 0, 0, 44, 62, 55, 0, 0, 0, 86, 254, 42 +31,1, 1, 0, 5, 0, 360, 1297, 375, 0, 50, 0, 1869, 12218, 2649, 0, 3187, 0, 424782, 818048, 346830, 0, 14848, 0, 3134865, 22410254, 4495068, 0, 45, 0, 2487, 8367, 2607, 0, 248, 0, 13899, 93445, 19620, 0, 5, 0, 360, 1297, 375, 0, 50, 0, 1869, 12218, 2649, 0, 7200, 0, 746799, 3557778, 2140017, 0, 130391, 0, 13935468, 73512363, 26076954, 0, 46, 0, 2616, 8657, 2733, 0, 433, 0, 15114, 100755, 22260 +30,1, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 6, 0 +10,2, 3, 1, 0, 0, 2, 15, 1, 0, 13, 0, 15, 135, 8, 10, 0, 0, 379, 43332, 125, 0, 617, 0, 5029, 68890, 1644, 3, 0, 0, 14, 107, 6, 0, 48, 0, 50, 580, 38, 1, 0, 0, 2, 15, 1, 0, 13, 0, 15, 135, 8, 8, 0, 0, 237, 53181, 13, 0, 25645, 0, 35224, 145849, 4652, 3, 0, 0, 12, 113, 3, 0, 125, 0, 60, 680, 39 +31,1, 3, 0, 0, 0, 1, 22, 3, 0, 12, 0, 9, 113, 12, 0, 0, 0, 2001, 10600, 833, 0, 20677, 0, 2323, 83214, 126086, 0, 0, 0, 10, 154, 22, 0, 42, 0, 46, 558, 81, 0, 0, 0, 1, 22, 3, 0, 12, 0, 9, 113, 12, 0, 0, 0, 2771, 45178, 787, 0, 36423, 0, 8578, 119968, 189146, 0, 0, 0, 11, 166, 21, 0, 94, 0, 61, 634, 86 +10,2, 1, 0, 0, 0, 231, 867, 195, 0, 18, 0, 249, 2923, 441, 0, 0, 0, 78285, 861087, 106026, 0, 5416, 0, 140037, 2603623, 613320, 0, 0, 0, 1152, 5007, 1266, 0, 74, 0, 1443, 21325, 3441, 0, 0, 0, 231, 867, 195, 0, 18, 0, 249, 2923, 441, 0, 0, 0, 177159, 2282982, 254988, 0, 40803, 0, 374898, 4959799, 1258254, 0, 0, 0, 1242, 5322, 1329, 0, 158, 0, 1725, 22549, 3534 +10,2, 2, 0, 0, 0, 382, 206, 440, 0, 12, 0, 436, 1181, 522, 0, 0, 0, 338440, 179608, 189778, 0, 3659, 0, 399948, 1294789, 530686, 0, 0, 0, 2860, 1566, 2832, 0, 44, 0, 3044, 8627, 3778, 0, 0, 0, 382, 206, 440, 0, 12, 0, 436, 1181, 522, 0, 0, 0, 2790152, 11499822, 10658090, 0, 4320, 0, 1070786, 2967304, 1520102, 0, 0, 0, 2942, 1640, 2970, 0, 82, 0, 3148, 9059, 3766 +31,1, 2, 0, 0, 0, 432, 283, 216, 0, 2, 0, 156, 1402, 250, 0, 0, 0, 241102, 203009, 117302, 0, 1, 0, 220198, 1767397, 230006, 0, 0, 0, 3118, 2046, 1534, 0, 1, 0, 1050, 10563, 1716, 0, 0, 0, 432, 283, 216, 0, 2, 0, 156, 1402, 250, 0, 0, 0, 459222, 359185, 452268, 0, 989, 0, 1623800, 5749235, 522196, 0, 0, 0, 3060, 1988, 1658, 0, 12, 0, 1034, 11016, 1834 +1,2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0 +1,2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2948, 5103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3514, 5190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 30, 0 +31,1, 1, 0, 0, 0, 252, 534, 192, 1, 18, 0, 470, 3052, 585, 0, 0, 0, 130728, 357018, 71310, 65, 2549, 0, 316636, 3793799, 884262, 0, 0, 0, 1584, 2796, 1065, 6, 79, 0, 2346, 22520, 4353, 0, 0, 0, 252, 534, 192, 1, 18, 0, 470, 3052, 585, 0, 0, 0, 2153148, 1191795, 145011, 3687, 27501, 0, 936858, 10172012, 1434768, 0, 0, 0, 1764, 2859, 1134, 11, 150, 0, 3001, 24179, 4539 +10,2, 3, 0, 0, 0, 1, 19, 0, 0, 7, 0, 6, 134, 13, 0, 0, 0, 123, 3485, 0, 0, 132, 0, 9069, 25398, 121797, 0, 0, 0, 6, 112, 0, 0, 14, 0, 58, 421, 74, 0, 0, 0, 1, 19, 0, 0, 7, 0, 6, 134, 13, 0, 0, 0, 506, 9007, 0, 0, 9836, 0, 6675, 36452, 130681, 0, 0, 0, 8, 126, 0, 0, 57, 0, 53, 489, 74 +1,2, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 42, 3, 0, 0, 0, 0, 1001, 0, 0, 0, 0, 0, 12992, 1443, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 179, 10, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 42, 3, 0, 0, 0, 0, 805, 0, 0, 0, 0, 0, 18942, 9651, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 181, 23 +11,2, 2, 0, 0, 0, 782, 266, 200, 0, 9, 0, 217, 801, 446, 0, 0, 0, 382170, 200916, 123106, 0, 502, 0, 218223, 632414, 509910, 0, 0, 0, 5802, 2138, 1344, 0, 21, 0, 1505, 5201, 3002, 0, 0, 0, 782, 266, 200, 0, 9, 0, 217, 801, 446, 0, 0, 0, 10882108, 10021028, 2095686, 0, 2770, 0, 494769, 1345960, 10609788, 0, 0, 0, 5962, 2222, 1376, 0, 56, 0, 1573, 5544, 3332 +11,2, 3, 0, 0, 0, 5, 5, 1, 1, 7, 0, 22, 72, 5, 0, 0, 0, 2633, 930, 591, 0, 620, 0, 6619, 119215, 5088, 0, 0, 0, 31, 29, 9, 0, 38, 0, 91, 326, 28, 0, 0, 0, 5, 5, 1, 1, 7, 0, 22, 72, 5, 0, 0, 0, 30595, 1902, 5796, 200, 19172, 0, 50011, 133536, 6676, 0, 0, 0, 44, 31, 12, 7, 67, 0, 126, 366, 29 +10,2, 1, 0, 0, 0, 186, 442, 144, 0, 6, 0, 225, 1569, 216, 0, 0, 0, 68880, 471894, 127671, 0, 171, 0, 660465, 1331091, 199572, 0, 0, 0, 807, 2593, 1002, 0, 14, 0, 1524, 10615, 1548, 0, 0, 0, 186, 442, 144, 0, 6, 0, 225, 1569, 216, 0, 0, 0, 175662, 724835, 615966, 0, 2123, 0, 862578, 2701621, 286509, 0, 0, 0, 975, 2679, 1017, 0, 42, 0, 1686, 11418, 1656 +10,2, 2, 0, 0, 0, 344, 224, 338, 0, 9, 0, 488, 896, 508, 0, 0, 0, 253386, 121866, 258638, 0, 87, 0, 437086, 573165, 429428, 0, 0, 0, 2426, 1736, 2426, 0, 13, 0, 2544, 5479, 3048, 0, 0, 0, 344, 224, 338, 0, 9, 0, 488, 896, 508, 0, 0, 0, 4537880, 2155564, 319858, 0, 5804, 0, 637346, 1807659, 755388, 0, 0, 0, 2578, 1742, 2442, 0, 62, 0, 2072, 5351, 2824 +1,2, 2, 0, 0, 0, 56, 28, 28, 0, 2, 0, 14, 98, 31, 0, 0, 0, 39880, 12980, 14752, 0, 1, 0, 13082, 146739, 69600, 0, 0, 0, 410, 222, 188, 0, 1, 0, 94, 687, 241, 0, 0, 0, 56, 28, 28, 0, 2, 0, 14, 98, 31, 0, 0, 0, 39482, 17192, 51050, 0, 1116, 0, 9946, 218394, 95366, 0, 0, 0, 380, 182, 206, 0, 16, 0, 84, 700, 263 +1,2, 1, 0, 0, 0, 24, 103, 27, 0, 6, 0, 63, 423, 99, 0, 0, 0, 17937, 69709, 18150, 0, 27, 0, 50496, 806610, 24249, 0, 0, 0, 111, 523, 180, 0, 9, 0, 441, 3006, 447, 0, 0, 0, 24, 103, 27, 0, 6, 0, 63, 423, 99, 0, 0, 0, 13137, 72302, 13839, 0, 57368, 0, 60501, 1344592, 90882, 0, 0, 0, 108, 526, 159, 0, 66, 0, 450, 3546, 522 +11,2, 1, 0, 0, 0, 111, 360, 153, 0, 4, 0, 117, 1887, 180, 0, 0, 0, 49440, 376752, 295707, 0, 2027, 0, 170955, 1564622, 229032, 0, 0, 0, 552, 2013, 1086, 0, 19, 0, 627, 13219, 1326, 0, 0, 0, 111, 360, 153, 0, 4, 0, 117, 1887, 180, 0, 0, 0, 45381, 502425, 1058184, 0, 3723, 0, 146010, 2608026, 490143, 0, 0, 0, 573, 1965, 1212, 0, 35, 0, 594, 13950, 1383 +1,2, 3, 0, 0, 0, 0, 12, 0, 0, 3, 0, 2, 71, 4, 0, 0, 0, 0, 2882, 0, 0, 2011, 0, 1368, 23137, 5091, 0, 0, 0, 0, 60, 0, 0, 16, 0, 10, 310, 28, 0, 0, 0, 0, 12, 0, 0, 3, 0, 2, 71, 4, 0, 0, 0, 0, 4409, 0, 0, 4924, 0, 1399, 70539, 5958, 0, 0, 0, 0, 75, 0, 0, 28, 0, 14, 355, 23 +11,2, 2, 0, 0, 0, 596, 334, 699, 0, 27, 0, 664, 2051, 984, 0, 0, 0, 469882, 275958, 378805, 0, 2167, 0, 654120, 2416553, 1045220, 0, 0, 0, 4206, 2378, 4001, 0, 65, 0, 4580, 13401, 6884, 0, 0, 0, 596, 334, 699, 0, 27, 0, 664, 2051, 984, 0, 0, 0, 11636992, 8538216, 6306017, 0, 68630, 0, 2427772, 7668218, 3064360, 0, 0, 0, 4484, 2588, 4301, 0, 184, 0, 4174, 14448, 7152 +11,2, 3, 0, 0, 0, 5, 12, 0, 0, 5, 0, 11, 109, 10, 0, 0, 0, 4880, 3384, 0, 0, 705, 0, 4210, 35184, 537, 0, 0, 0, 43, 73, 0, 0, 28, 0, 44, 334, 24, 0, 0, 0, 5, 12, 0, 0, 5, 0, 11, 109, 10, 0, 0, 0, 4569, 2474, 0, 0, 29972, 0, 18201, 36911, 2179, 0, 0, 0, 40, 78, 0, 0, 53, 0, 55, 330, 31 +1,2, 2, 0, 0, 0, 150, 92, 84, 0, 2, 0, 58, 397, 148, 0, 0, 0, 63464, 82596, 37414, 0, 1190, 0, 37398, 373719, 410978, 0, 0, 0, 936, 704, 532, 0, 16, 0, 380, 2814, 1058, 0, 0, 0, 150, 92, 84, 0, 2, 0, 58, 397, 148, 0, 0, 0, 67488, 216422, 111420, 0, 5299, 0, 123620, 694261, 906288, 0, 0, 0, 874, 752, 532, 0, 21, 0, 402, 2927, 1246 +1,2, 1, 0, 0, 0, 78, 363, 96, 0, 8, 0, 207, 1436, 414, 0, 0, 0, 31839, 185346, 75468, 0, 94, 0, 169176, 1427814, 428583, 0, 0, 0, 321, 2163, 522, 0, 15, 0, 1578, 10641, 3111, 0, 0, 0, 78, 363, 96, 0, 8, 0, 207, 1436, 414, 0, 0, 0, 27480, 1191345, 1026495, 0, 23286, 0, 678015, 2474767, 1449945, 0, 0, 0, 303, 2370, 600, 0, 63, 0, 1650, 11114, 3348 +11,2, 1, 0, 0, 0, 333, 1554, 276, 0, 14, 0, 249, 3371, 441, 0, 0, 0, 333189, 1526979, 530361, 0, 2929, 0, 115461, 2938925, 599745, 0, 0, 0, 2181, 11127, 2022, 0, 52, 0, 1464, 22216, 3285, 0, 0, 0, 333, 1554, 276, 0, 14, 0, 249, 3371, 441, 0, 0, 0, 756480, 3703635, 1058247, 0, 21343, 0, 424674, 6852989, 1145460, 0, 0, 0, 2319, 11478, 2127, 0, 120, 0, 1614, 25112, 3378 +11,2, 2, 0, 0, 0, 32, 36, 58, 0, 3, 0, 36, 146, 124, 0, 0, 0, 13280, 29314, 19086, 0, 71, 0, 25338, 111366, 299822, 0, 0, 0, 222, 236, 358, 0, 8, 0, 270, 936, 850, 0, 0, 0, 32, 36, 58, 0, 3, 0, 36, 146, 124, 0, 0, 0, 32874, 37402, 103992, 0, 4256, 0, 96546, 178452, 321002, 0, 0, 0, 222, 226, 340, 0, 26, 0, 318, 1037, 878 +1,2, 3, 0, 0, 0, 3, 14, 2, 0, 7, 0, 6, 101, 7, 0, 0, 0, 956, 8125, 569, 0, 7, 0, 5486, 88662, 4356, 0, 0, 0, 20, 102, 15, 0, 3, 0, 40, 476, 26, 0, 0, 0, 3, 14, 2, 0, 7, 0, 6, 101, 7, 0, 0, 0, 706, 8306, 305, 0, 20005, 0, 8582, 125805, 19710, 0, 0, 0, 16, 107, 13, 0, 50, 0, 49, 585, 28 +12,2, 1, 0, 0, 0, 480, 1611, 549, 0, 39, 0, 501, 5797, 1098, 0, 0, 0, 436494, 1324308, 663825, 0, 7409, 0, 454134, 5199690, 1581111, 0, 0, 0, 2889, 10884, 3897, 0, 158, 0, 3288, 41542, 8235, 0, 0, 0, 480, 1611, 549, 0, 39, 0, 501, 5797, 1098, 0, 0, 0, 957366, 3133371, 1268589, 0, 123744, 0, 2022165, 12480144, 2823045, 0, 0, 0, 3054, 11082, 4110, 0, 348, 0, 3801, 44707, 8667 +12,2, 2, 0, 0, 0, 438, 330, 444, 0, 7, 0, 770, 1345, 861, 0, 0, 0, 305922, 218656, 204172, 0, 3757, 0, 750454, 1442851, 650101, 0, 0, 0, 3160, 2342, 2758, 0, 19, 0, 4910, 8571, 5591, 0, 0, 0, 438, 330, 444, 0, 7, 0, 770, 1345, 861, 0, 0, 0, 496570, 10353604, 902838, 0, 4802, 0, 10215676, 3085270, 4518650, 0, 0, 0, 3248, 2598, 2868, 0, 42, 0, 5074, 9138, 5731 +12,2, 3, 0, 0, 0, 14, 10, 1, 0, 7, 0, 23, 186, 17, 0, 0, 0, 16763, 996, 213, 0, 701, 0, 4162, 78978, 127924, 0, 0, 0, 119, 58, 7, 0, 28, 0, 63, 628, 57, 0, 0, 0, 14, 10, 1, 0, 7, 0, 23, 186, 17, 0, 0, 0, 13307, 2110, 554, 0, 13452, 0, 4656, 116281, 125533, 0, 0, 0, 110, 66, 9, 0, 59, 0, 84, 691, 89 +11,2, 3, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4, 19, 1, 0, 0, 0, 0, 95, 0, 0, 130, 0, 60, 446, 149, 0, 0, 0, 0, 6, 0, 0, 12, 0, 5, 21, 7, 0, 0, 0, 0, 1, 0, 0, 2, 0, 4, 19, 1, 0, 0, 0, 0, 107, 0, 0, 7523, 0, 6, 2528, 17, 0, 0, 0, 0, 6, 0, 0, 22, 0, 2, 28, 4 +1,2, 1, 0, 0, 0, 76, 186, 78, 0, 2, 0, 156, 1033, 219, 0, 0, 0, 45902, 204954, 81237, 0, 65, 0, 231795, 2045638, 472398, 0, 0, 0, 440, 1128, 471, 0, 6, 0, 1017, 8265, 1788, 0, 0, 0, 76, 186, 78, 0, 2, 0, 156, 1033, 219, 0, 0, 0, 59533, 330057, 92025, 0, 3693, 0, 295110, 3006840, 6641223, 0, 0, 0, 475, 1152, 423, 0, 14, 0, 1104, 8452, 1917 +1,2, 2, 0, 0, 0, 287, 100, 152, 0, 3, 0, 74, 251, 94, 0, 0, 0, 260545, 100846, 135168, 0, 6, 0, 30784, 266488, 98722, 0, 0, 0, 2161, 778, 1086, 0, 2, 0, 404, 1630, 744, 0, 0, 0, 287, 100, 152, 0, 3, 0, 74, 251, 94, 0, 0, 0, 1520904, 109188, 1044098, 0, 18722, 0, 96162, 361138, 179880, 0, 0, 0, 2165, 754, 1084, 0, 24, 0, 442, 1804, 794 +11,2, 1, 0, 0, 0, 51, 181, 48, 0, 1, 0, 36, 677, 66, 0, 0, 0, 43560, 110905, 51441, 0, 149, 0, 12339, 389560, 80652, 0, 0, 0, 303, 1096, 342, 0, 7, 0, 165, 4109, 465, 0, 0, 0, 51, 181, 48, 0, 1, 0, 36, 677, 66, 0, 0, 0, 50400, 385135, 49236, 0, 899, 0, 17154, 852598, 91956, 0, 0, 0, 345, 1190, 357, 0, 9, 0, 228, 4920, 489 +14,1, 1, 0, 1, 0, 291, 932, 237, 0, 27, 0, 213, 5053, 888, 0, 279, 0, 132072, 735201, 479475, 0, 13034, 0, 251091, 7770346, 1407144, 0, 8, 0, 1629, 6013, 1902, 0, 87, 0, 1761, -2147445847, 7125, 0, 1, 0, 291, 932, 237, 0, 27, 0, 213, 5053, 888, 0, 82, 0, 240468, 1424960, 548811, 0, 93706, 0, 991329, 14307679, 2791926, 0, 6, 0, 1701, 5907, 1857, 0, 204, 0, 1854, -2147444081, 7182 +13,1, 1, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 37, 3, 0, 0, 0, 0, 168, 0, 0, 6244, 0, 0, 33007, 1386, 0, 0, 0, 0, 27, 0, 0, 21, 0, 0, 266, 24, 0, 0, 0, 0, 6, 0, 0, 3, 0, 0, 37, 3, 0, 0, 0, 0, 195, 0, 0, 26916, 0, 0, 225463, 1203, 0, 0, 0, 0, 24, 0, 0, 30, 0, 0, 351, 24 +15,1, 1, 0, 0, 0, 180, 546, 126, 0, 21, 0, 120, 2635, 321, 0, 0, 0, 80934, 337830, 77211, 0, 6360, 0, 358170, 2558605, 876603, 0, 0, 0, 984, 3450, 882, 0, 46, 0, 990, 18811, 2643, 0, 0, 0, 180, 546, 126, 0, 21, 0, 120, 2635, 321, 0, 0, 0, 150807, 752118, 141204, 0, 102611, 0, 390264, 5139889, 1448361, 0, 0, 0, 1032, 3498, 915, 0, 165, 0, 972, 20249, 2784 +15,1, 3, 0, 0, 0, 6, 22, 2, 0, 11, 0, 4, 95, 17, 0, 0, 0, 2328, 18162, 450, 0, 695, 0, 2012, 67092, 3254, 0, 0, 0, 45, 157, 13, 0, 25, 0, 15, 511, 72, 0, 0, 0, 6, 22, 2, 0, 11, 0, 4, 95, 17, 0, 0, 0, 2722, 24394, 280, 0, 6784, 0, 931, 110199, 10907, 0, 0, 0, 48, 156, 12, 0, 79, 0, 24, 599, 58 +14,1, 3, 0, 0, 0, 7, 66, 0, 0, 18, 0, 18, 120, 9, 0, 0, 0, 1111, 20954, 0, 0, 1079, 0, 7379, 109701, 119378, 0, 0, 0, 42, 442, 0, 0, 25, 0, 79, 697, 70, 0, 0, 0, 7, 66, 0, 0, 18, 0, 18, 120, 9, 0, 0, 0, 2897, 45345, 0, 0, 10798, 0, 32152, 149560, 132099, 0, 0, 0, 41, 443, 0, 0, 135, 0, 78, 807, 75 +15,1, 2, 3, 0, 0, 618, 492, 352, 0, 43, 0, 546, 2101, 862, 2434, 0, 0, 658104, 732528, 1004634, 0, 7956, 0, 827496, 3087068, 1329980, 26, 0, 0, 4810, 4076, 2876, 0, 107, 0, 3788, 14711, 6818, 3, 0, 0, 618, 492, 352, 0, 43, 0, 546, 2101, 862, 15979, 0, 0, 11138257, 3160922, 1694066, 0, 90263, 0, 1969084, 8600957, 4992870, 28, 0, 0, 5002, 4078, 2922, 0, 339, 0, 4444, 17041, 7340 +13,1, 2, 0, 0, 0, 9, 2, 0, 0, 0, 0, 8, 14, 10, 0, 0, 0, 6447, 1644, 0, 0, 0, 0, 16466, 3558, 9136, 0, 0, 0, 76, 18, 0, 0, 0, 0, 76, 84, 76, 0, 0, 0, 9, 2, 0, 0, 0, 0, 8, 14, 10, 0, 0, 0, 10535, 624, 0, 0, 0, 0, 11364, 26046, 183758, 0, 0, 0, 86, 16, 0, 0, 0, 0, 72, 118, 98 +14,1, 2, 1, 11, 0, 525, 459, 497, 0, 96, 0, 408, 3178, 1342, 1117, 5517, 0, 320755, 478147, 534606, 0, 17672, 0, 558684, 3256388, 1181062, 10, 79, 0, 3812, 3407, 3622, 0, 278, 0, 2972, 21118, 9310, 1, 11, 0, 525, 459, 497, 0, 96, 0, 408, 3178, 1342, 872, 114403, 0, 744300, 2463267, 1386566, 0, 216301, 0, 1085828, 9311017, 13080986, 9, 92, 0, 3851, 3714, 3772, 0, 718, 0, 3218, 24370, 10142 +13,1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13 +1,2, 3, 0, 0, 0, 0, 7, 1, 0, 0, 0, 3, 43, 3, 0, 0, 0, 0, 5608, 116, 0, 0, 0, 1956, 23726, 1047, 0, 0, 0, 0, 56, 6, 0, 0, 0, 10, 220, 10, 0, 0, 0, 0, 7, 1, 0, 0, 0, 3, 43, 3, 0, 0, 0, 0, 5599, 14, 0, 0, 0, 591, 43644, 263, 0, 0, 0, 0, 49, 3, 0, 0, 0, 9, 235, 10 +13,2, 1, 0, 2, 0, 279, 1075, 297, 0, 21, 0, 291, 4111, 633, 0, 0, 0, 144630, 760377, 196530, 0, 4684, 0, 165573, 5900535, 797034, 0, 0, 0, 1689, 7038, 1923, 0, 69, 0, 1899, 29706, 4572, 0, 2, 0, 279, 1075, 297, 0, 21, 0, 291, 4111, 633, 0, 175, 0, 260919, 1292591, 286146, 0, 52027, 0, 342519, 10371775, 1774596, 0, 10, 0, 1746, 6989, 2058, 0, 163, 0, 2019, 30816, 4941 +13,2, 2, 0, 0, 0, 403, 358, 312, 0, 20, 0, 476, 1110, 680, 0, 0, 0, 229568, 264762, 503286, 0, 132, 0, 640008, 1361838, 1059784, 0, 0, 0, 2895, 2696, 2152, 0, 19, 0, 2794, 6951, 4692, 0, 0, 0, 403, 358, 312, 0, 20, 0, 476, 1110, 680, 0, 0, 0, 4616352, 4473138, 1801296, 0, 2965, 0, 1002856, 2714061, 5100198, 0, 0, 0, 2941, 2744, 2220, 0, 110, 0, 2852, 7760, 4990 +12,2, 1, 0, 0, 0, 186, 610, 180, 0, 2, 0, 72, 1741, 303, 0, 0, 0, 66651, 377432, 117771, 0, 130, 0, 39801, 2899805, 526374, 0, 0, 0, 831, 3548, 1092, 0, 12, 0, 453, 12114, 2469, 0, 0, 0, 186, 610, 180, 0, 2, 0, 72, 1741, 303, 0, 0, 0, 191184, 749380, 171825, 0, 7546, 0, 130590, 5436362, 841119, 0, 0, 0, 987, 3718, 1191, 0, 22, 0, 561, 13265, 2541 +12,2, 2, 0, 0, 0, 294, 138, 230, 0, 3, 0, 287, 643, 418, 0, 0, 0, 272146, 120016, 132654, 0, 14, 0, 234094, 678478, 529062, 0, 0, 0, 2094, 1050, 1512, 0, 6, 0, 1692, 3976, 2584, 0, 0, 0, 294, 138, 230, 0, 3, 0, 287, 643, 418, 0, 0, 0, 469832, 189984, 604208, 0, 19916, 0, 264077, 2017812, 808028, 0, 0, 0, 2158, 1082, 1540, 0, 31, 0, 1552, 4167, 2616 +12,2, 3, 0, 0, 0, 2, 8, 2, 0, 5, 0, 8, 136, 5, 0, 0, 0, 2987, 1128, 674, 0, 132, 0, 336, 26348, 1746, 0, 0, 0, 17, 42, 15, 0, 14, 0, 23, 385, 28, 0, 0, 0, 2, 8, 2, 0, 5, 0, 8, 136, 5, 0, 0, 0, 6961, 875, 1144, 0, 9557, 0, 2265, 40615, 4982, 0, 0, 0, 18, 49, 18, 0, 44, 0, 33, 401, 38 +13,2, 3, 0, 1, 0, 3, 13, 2, 0, 11, 0, 10, 158, 4, 0, 291, 0, 5809, 1605, 1164, 0, 997, 0, 9414, 66810, 2023, 0, 8, 0, 30, 78, 17, 0, 35, 0, 65, 622, 31, 0, 1, 0, 3, 13, 2, 0, 11, 0, 10, 158, 4, 0, 134, 0, 29143, 1832, 1990, 0, 13400, 0, 8026, 313163, 5205, 0, 7, 0, 33, 79, 17, 0, 81, 0, 65, 649, 37 +1,2, 1, 0, 0, 0, 36, 696, 48, 0, 2, 0, 54, 530, 114, 0, 0, 0, 3009, 36666, 19248, 0, 5328, 0, 9999, 727951, 329223, 0, 0, 0, 150, 1197, 261, 0, 12, 0, 201, 4073, 945, 0, 0, 0, 36, 696, 48, 0, 2, 0, 54, 530, 114, 0, 0, 0, 4050, 32031, 1587057, 0, 5019, 0, 16368, 1433229, 419940, 0, 0, 0, 165, 1419, 321, 0, 19, 0, 273, 4248, 1005 +1,2, 2, 0, 0, 0, 117, 70, 18, 0, 1, 0, 52, 119, 36, 0, 0, 0, 48077, 34012, 39238, 0, 0, 0, 45334, 89970, 73634, 0, 0, 0, 803, 536, 126, 0, 0, 0, 394, 754, 270, 0, 0, 0, 117, 70, 18, 0, 1, 0, 52, 119, 36, 0, 0, 0, 140379, 246652, 42420, 0, 13, 0, 226314, 215103, 71004, 0, 0, 0, 878, 636, 124, 0, 3, 0, 400, 713, 262 +1,2, 3, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 19, 1, 0, 0, 0, 0, 7331, 0, 0, 0, 0, 125, 12248, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 6, 113, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 19, 1, 0, 0, 0, 0, 5901, 0, 0, 0, 0, 195, 10340, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 7, 125, 0 +13,2, 1, 0, 0, 0, 291, 742, 240, 0, 27, 0, 207, 3798, 384, 0, 0, 0, 112836, 340241, 223503, 0, 2811, 0, 116091, 3648528, 564432, 0, 0, 0, 1554, 4050, 1578, 0, 44, 0, 1182, 26368, 2793, 0, 0, 0, 291, 742, 240, 0, 27, 0, 207, 3798, 384, 0, 0, 0, 223482, 736568, 392784, 0, 48761, 0, 369672, 12451036, 1247670, 0, 0, 0, 1743, 4308, 1677, 0, 188, 0, 1374, 28642, 3159 +14,2, 1, 0, 0, 0, 84, 282, 96, 0, 2, 0, 88, 1414, 177, 0, 0, 0, 135936, 178881, 51414, 0, 734, 0, 50949, 1045510, 389973, 0, 0, 0, 258, 1581, 510, 0, 14, 0, 525, 9559, 1428, 0, 0, 0, 84, 282, 96, 0, 2, 0, 88, 1414, 177, 0, 0, 0, 156009, 286080, 71157, 0, 2175, 0, 215226, 1790214, 1112805, 0, 0, 0, 354, 1623, 471, 0, 15, 0, 551, 10545, 1650 +13,2, 2, 0, 0, 0, 402, 238, 324, 0, 7, 0, 758, 1045, 639, 0, 0, 0, 182778, 220738, 234984, 0, 3902, 0, 145642, 894848, 351259, 0, 0, 0, 2690, 1794, 2112, 0, 31, 0, 2468, 5545, 3201, 0, 0, 0, 402, 238, 324, 0, 7, 0, 758, 1045, 639, 0, 0, 0, 8430292, 3056288, 1441120, 0, 2223, 0, 212206, 2049081, 563480, 0, 0, 0, 2782, 1844, 2192, 0, 43, 0, 1820, 5177, 2835 +14,2, 3, 0, 0, 0, 0, 8, 0, 0, 8, 0, 1, 84, 0, 0, 0, 0, 0, 2032, 0, 0, 66, 0, 0, 21368, 0, 0, 0, 0, 0, 50, 0, 0, 7, 0, 0, 177, 0, 0, 0, 0, 0, 8, 0, 0, 8, 0, 1, 84, 0, 0, 0, 0, 0, 7820, 0, 0, 5377, 0, 0, 65840, 0, 0, 0, 0, 0, 61, 0, 0, 49, 0, 0, 463, 0 +13,2, 3, 0, 0, 0, 2, 20, 1, 0, 17, 0, 13, 164, 10, 0, 0, 0, 4158, 2962, 211, 0, 393, 0, 0, 42680, 5876, 0, 0, 0, 21, 111, 7, 0, 39, 0, 0, 480, 34, 0, 0, 0, 2, 20, 1, 0, 17, 0, 13, 164, 10, 0, 0, 0, 2837, 3901, 181, 0, 26279, 0, 19, 57853, 3533, 0, 0, 0, 20, 120, 7, 0, 132, 0, 7, 552, 30 +1,2, 1, 0, 0, 0, 27, 33, 9, 0, 1, 0, 12, 159, 54, 0, 0, 0, 12861, 14511, 810, 0, 6, 0, 582, 210207, 40323, 0, 0, 0, 162, 135, 36, 0, 2, 0, 60, 1262, 408, 0, 0, 0, 27, 33, 9, 0, 1, 0, 12, 159, 54, 0, 0, 0, 11388, 28476, 32469, 0, 211, 0, 144, 226487, 38490, 0, 0, 0, 156, 141, 69, 0, 7, 0, 39, 1293, 426 +1,2, 2, 0, 0, 0, 34, 18, 20, 0, 0, 0, 13, 54, 28, 0, 0, 0, 12148, 7092, 11816, 0, 0, 0, 9497, 80262, 54296, 0, 0, 0, 234, 110, 142, 0, 0, 0, 101, 458, 212, 0, 0, 0, 34, 18, 20, 0, 0, 0, 13, 54, 28, 0, 0, 0, 11442, 13182, 73296, 0, 0, 0, 12743, 72832, 53348, 0, 0, 0, 206, 124, 138, 0, 0, 0, 102, 462, 196 +14,2, 2, 0, 0, 0, 84, 68, 98, 0, 4, 0, 120, 329, 186, 0, 0, 0, 69250, 33644, 103014, 0, 1, 0, 33186, 181992, 81038, 0, 0, 0, 628, 494, 698, 0, 1, 0, 480, 1849, 1002, 0, 0, 0, 84, 68, 98, 0, 4, 0, 120, 329, 186, 0, 0, 0, 141122, 44572, 131826, 0, 1125, 0, 55518, 744693, 151106, 0, 0, 0, 644, 458, 698, 0, 22, 0, 446, 2036, 970 +15,2, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 2, 2, 0, 0, 0, 0, 0, 3728, 0, 0, 0, 79, 66, 48, 0, 0, 0, 0, 0, 20, 0, 0, 0, 11, 10, 8, 0, 0, 0, 0, 0, 2, 0, 0, 0, 3, 2, 2, 0, 0, 0, 0, 0, 2814, 0, 0, 0, 27, 10, 26, 0, 0, 0, 0, 0, 20, 0, 0, 0, 6, 4, 6 +14,2, 1, 0, 3, 0, 231, 2433, 237, 0, 23, 0, 267, 4016, 507, 0, 83, 0, 89946, 1687609, 641502, 0, 79, 0, 235425, 4926931, 948657, 0, 12, 0, 1332, 17286, 1476, 0, 25, 0, 1707, 29321, 3870, 0, 3, 0, 231, 2433, 237, 0, 23, 0, 267, 4016, 507, 0, 159, 0, 554586, 20782752, 1955727, 0, 9185, 0, 559977, 13038698, 2204430, 0, 15, 0, 1452, 16713, 1641, 0, 154, 0, 1851, 30683, 4122 +14,2, 3, 0, 1, 0, 3, 28, 2, 0, 7, 0, 3, 245, 14, 0, 1926, 0, 422, 10625, 338, 0, 284, 0, 0, 238453, 113934, 0, 10, 0, 19, 191, 13, 0, 10, 0, 0, 776, 66, 0, 1, 0, 3, 28, 2, 0, 7, 0, 3, 245, 14, 0, 1082, 0, 558, 12391, 357, 0, 2578, 0, 1, 606940, 111742, 0, 10, 0, 21, 197, 13, 0, 45, 0, 1, 918, 78 +14,2, 2, 0, 1, 0, 688, 353, 622, 1, 17, 0, 861, 1529, 921, 0, 128, 0, 348330, 191450, 206968, 886, 9732, 0, 285408, 911400, 444608, 0, 7, 0, 4738, 2439, 3696, 9, 75, 0, 3473, 8681, 4287, 0, 1, 0, 688, 353, 622, 1, 17, 0, 861, 1529, 921, 0, 29, 0, 1485626, 453035, 830430, 3166, 31581, 0, 2403472, 8433835, 1945465, 0, 4, 0, 4938, 2554, 3752, 11, 138, 0, 2727, 9148, 4436 +15,2, 3, 0, 0, 0, 2, 16, 0, 0, 11, 0, 26, 250, 16, 0, 0, 0, 661, 14049, 0, 0, 2144, 0, 23771, 322697, 155873, 0, 0, 0, 13, 90, 0, 0, 31, 0, 166, 856, 151, 0, 0, 0, 2, 16, 0, 0, 11, 0, 26, 250, 16, 0, 0, 0, 813, 14387, 0, 0, 11609, 0, 36908, 481941, 327962, 0, 0, 0, 11, 95, 0, 0, 77, 0, 175, 902, 189 +15,2, 2, 0, 0, 0, 727, 316, 558, 0, 2, 0, 872, 1330, 1022, 0, 0, 0, 378855, 254332, 215588, 0, 0, 0, 391944, 1175968, 924098, 0, 0, 0, 4786, 2426, 3384, 0, 0, 0, 3530, 8318, 5900, 0, 0, 0, 727, 316, 558, 0, 2, 0, 872, 1330, 1022, 0, 0, 0, 1118494, 490324, 886366, 0, 188, 0, 789094, 2552624, 1958562, 0, 0, 0, 4832, 2406, 3452, 0, 11, 0, 2948, 8663, 5622 +1,2, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 146, 0, 0, 0, 0, 0, 2413, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 220, 0, 0, 0, 0, 0, 9105, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 56, 0 +14,2, 1, 0, 0, 0, 72, 555, 42, 0, 0, 0, 39, 460, 108, 0, 0, 0, 44349, 3043977, 15435, 0, 0, 0, 16743, 228036, 45783, 0, 0, 0, 513, 4317, 273, 0, 0, 0, 216, 2868, 618, 0, 0, 0, 72, 555, 42, 0, 0, 0, 39, 460, 108, 0, 0, 0, 104256, 6269292, 66300, 0, 0, 0, 152880, 540559, 228369, 0, 0, 0, 537, 4581, 309, 0, 0, 0, 261, 2657, 792 +14,2, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 22, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 94, 4535, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 6, 47, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 22, 0, 0, 0, 0, 0, 364, 0, 0, 0, 0, 27, 5267, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 5, 53, 0 +1,2, 2, 0, 0, 0, 24, 16, 14, 0, 0, 0, 6, 43, 12, 0, 0, 0, 11078, 11492, 2288, 0, 0, 0, 582, 23872, 10118, 0, 0, 0, 158, 128, 66, 0, 0, 0, 28, 282, 84, 0, 0, 0, 24, 16, 14, 0, 0, 0, 6, 43, 12, 0, 0, 0, 14170, 33086, 1328, 0, 0, 0, 1032, 30378, 17942, 0, 0, 0, 152, 120, 56, 0, 0, 0, 22, 300, 112 +1,2, 1, 0, 0, 0, 15, 24, 18, 0, 1, 0, 6, 152, 28, 0, 0, 0, 1059, 2478, 4191, 0, 65, 0, 1143, 148339, 37896, 0, 0, 0, 57, 81, 78, 0, 6, 0, 39, 1182, 202, 0, 0, 0, 15, 24, 18, 0, 1, 0, 6, 152, 28, 0, 0, 0, 486, 27057, 6408, 0, 3688, 0, 2469, 332207, 73438, 0, 0, 0, 42, 123, 87, 0, 11, 0, 42, 1171, 206 +14,2, 2, 0, 0, 0, 59, 38, 52, 0, 1, 0, 28, 151, 76, 0, 0, 0, 38579, 56086, 35474, 0, 74, 0, 77820, 76670, 78664, 0, 0, 0, 476, 320, 400, 0, 6, 0, 212, 968, 568, 0, 0, 0, 59, 38, 52, 0, 1, 0, 28, 151, 76, 0, 0, 0, 75286, 158600, 49436, 0, 208, 0, 70078, 121988, 120792, 0, 0, 0, 503, 330, 426, 0, 7, 0, 188, 1061, 552 +15,2, 1, 0, 1, 0, 348, 689, 189, 0, 7, 0, 117, 2007, 381, 0, 29, 0, 182571, 340453, 61422, 0, 2257, 0, 40620, 2028147, 903627, 0, 4, 0, 2013, 3479, 906, 0, 35, 0, 507, 13880, 3153, 0, 1, 0, 348, 689, 189, 0, 7, 0, 117, 2007, 381, 0, 55, 0, 318702, 440837, 64101, 0, 5329, 0, 168615, 2794046, 1487415, 0, 5, 0, 2124, 3706, 978, 0, 48, 0, 564, 13847, 3321 +15,2, 3, 0, 0, 0, 0, 5, 1, 0, 1, 0, 5, 40, 2, 0, 0, 0, 0, 231, 896, 0, 194, 0, 0, 11559, 135, 0, 0, 0, 0, 26, 9, 0, 7, 0, 0, 127, 9, 0, 0, 0, 0, 5, 1, 0, 1, 0, 5, 40, 2, 0, 0, 0, 0, 462, 113, 0, 56, 0, 2, 17059, 34, 0, 0, 0, 0, 30, 6, 0, 5, 0, 2, 155, 7 +17,2, 2, 0, 0, 0, 276, 122, 259, 0, 11, 0, 118, 539, 186, 0, 0, 0, 116024, 97186, 130559, 0, 10, 0, 194656, 489158, 147332, 0, 0, 0, 1894, 918, 1573, 0, 5, 0, 872, 3465, 1316, 0, 0, 0, 276, 122, 259, 0, 11, 0, 118, 539, 186, 0, 0, 0, 934040, 145998, 4240784, 0, 54024, 0, 299494, 741280, 208896, 0, 0, 0, 1926, 920, 1672, 0, 75, 0, 890, 3781, 1410 +16,2, 2, 0, 0, 0, 903, 404, 824, 0, 1, 0, 454, 1493, 590, 0, 0, 0, 608026, 271474, 329392, 0, 0, 0, 508812, 1641542, 573124, 0, 0, 0, 6093, 2968, 4952, 0, 0, 0, 3364, 10408, 4152, 0, 0, 0, 903, 404, 824, 0, 1, 0, 454, 1493, 590, 0, 0, 0, 4727115, 938698, 4009832, 0, 18, 0, 4011270, 3049898, 894434, 0, 0, 0, 6150, 2946, 4934, 0, 4, 0, 3516, 10864, 4216 +1,2, 3, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 20, 2, 0, 0, 0, 0, 6200, 0, 0, 1863, 0, 0, 7487, 0, 0, 0, 0, 0, 33, 0, 0, 10, 0, 0, 95, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 20, 2, 0, 0, 0, 0, 14834, 0, 0, 1387, 0, 0, 9682, 0, 0, 0, 0, 0, 37, 0, 0, 10, 0, 0, 103, 0 +15,2, 2, 0, 0, 0, 84, 28, 52, 0, 4, 0, 40, 147, 74, 0, 0, 0, 47694, 25106, 23002, 0, 1, 0, 13694, 80926, 109150, 0, 0, 0, 620, 226, 364, 0, 1, 0, 220, 768, 520, 0, 0, 0, 84, 28, 52, 0, 4, 0, 40, 147, 74, 0, 0, 0, 74652, 511812, 110884, 0, 1362, 0, 25204, 125571, 141650, 0, 0, 0, 648, 270, 382, 0, 27, 0, 232, 847, 514 +16,2, 1, 0, 0, 0, 315, 858, 261, 0, 8, 0, 138, 1700, 315, 0, 0, 0, 258177, 505527, 125574, 0, 471, 0, 76395, 1475305, 350946, 0, 0, 0, 1269, 4191, 1320, 0, 27, 0, 726, 10769, 2268, 0, 0, 0, 315, 858, 261, 0, 8, 0, 138, 1700, 315, 0, 0, 0, 361935, 779094, 508776, 0, 4776, 0, 138690, 5673408, 460500, 0, 0, 0, 1518, 4311, 1380, 0, 63, 0, 906, 11405, 2268 +17,2, 1, 0, 0, 0, 108, 373, 156, 0, 1, 0, 57, 1796, 321, 0, 0, 0, 27822, 218113, 72642, 0, 2, 0, 41886, 1723489, 268581, 0, 0, 0, 486, 2197, 774, 0, 1, 0, 450, 12929, 2361, 0, 0, 0, 108, 373, 156, 0, 1, 0, 57, 1796, 321, 0, 0, 0, 28809, 1391963, 78801, 0, 598, 0, 121302, 2690549, 386343, 0, 0, 0, 546, 2287, 807, 0, 9, 0, 495, 13317, 2316 +16,2, 3, 0, 0, 0, 4, 38, 1, 0, 24, 0, 6, 283, 39, 0, 0, 0, 2173, 6946, 217, 0, 12, 0, 2974, 86503, 114879, 0, 0, 0, 33, 226, 7, 0, 8, 0, 33, 784, 116, 0, 0, 0, 4, 38, 1, 0, 24, 0, 6, 283, 39, 0, 0, 0, 1619, 21982, 575, 0, 26141, 0, 2833, 125645, 112484, 0, 0, 0, 31, 247, 9, 0, 159, 0, 31, 965, 114 +1,2, 1, 0, 0, 0, 27, 72, 15, 0, 0, 0, 99, 400, 75, 0, 0, 0, 9699, 25323, 2694, 0, 0, 0, 72852, 494439, 58725, 0, 0, 0, 108, 291, 69, 0, 0, 0, 687, 2766, 534, 0, 0, 0, 27, 72, 15, 0, 0, 0, 99, 400, 75, 0, 0, 0, 4920, 1592274, 3084, 0, 0, 0, 299499, 3389888, 123360, 0, 0, 0, 105, 327, 63, 0, 0, 0, 771, 2956, 594 +1,2, 2, 0, 0, 0, 42, 28, 30, 0, 0, 0, 8, 85, 20, 0, 0, 0, 7600, 9392, 14490, 0, 0, 0, 2214, 113674, 30232, 0, 0, 0, 240, 176, 212, 0, 0, 0, 46, 617, 176, 0, 0, 0, 42, 28, 30, 0, 0, 0, 8, 85, 20, 0, 0, 0, 6364, 6988, 27400, 0, 0, 0, 2072, 114911, 33332, 0, 0, 0, 228, 180, 204, 0, 0, 0, 52, 621, 186 +17,2, 3, 1, 0, 0, 2, 13, 1, 0, 9, 0, 8, 98, 4, 1176, 0, 0, 270, 19526, 212, 0, 67, 0, 110, 18347, 4705, 10, 0, 0, 13, 76, 7, 0, 8, 0, 19, 247, 29, 1, 0, 0, 2, 13, 1, 0, 9, 0, 8, 98, 4, 1108, 0, 0, 226, 26141, 42, 0, 6389, 0, 140, 28636, 4870, 10, 0, 0, 11, 91, 5, 0, 58, 0, 16, 390, 31 +15,2, 1, 0, 0, 0, 48, 111, 49, 0, 2, 0, 15, 328, 36, 0, 0, 0, 22638, 55527, 14733, 0, 86, 0, 14781, 217087, 28566, 0, 0, 0, 165, 621, 171, 0, 10, 0, 54, 2174, 276, 0, 0, 0, 48, 111, 49, 0, 2, 0, 15, 328, 36, 0, 0, 0, 20982, 60036, 11243, 0, 4034, 0, 16392, 303772, 27621, 0, 0, 0, 192, 645, 221, 0, 18, 0, 90, 2340, 258 +18,2, 3, 0, 0, 0, 2, 7, 1, 0, 9, 0, 14, 113, 5, 0, 0, 0, 168, 421, 23, 0, 378, 0, 11464, 24642, 228, 0, 0, 0, 12, 38, 4, 0, 16, 0, 62, 341, 13, 0, 0, 0, 2, 7, 1, 0, 9, 0, 14, 113, 5, 0, 0, 0, 16, 682, 43, 0, 3065, 0, 11260, 82413, 159, 0, 0, 0, 6, 42, 5, 0, 63, 0, 61, 391, 14 +1,2, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 629, 0, 0, 0, 0, 0, 1539, 2768, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 18, 25, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 3, 0, 0, 0, 0, 1699, 0, 0, 0, 0, 0, 3354, 64048, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 20, 29 +17,2, 2, 0, 0, 0, 749, 364, 862, 0, 18, 0, 450, 970, 468, 0, 0, 0, 440827, 248138, 452830, 0, 11652, 0, 313486, 733660, 560670, 0, 0, 0, 5156, 2602, 4532, 0, 58, 0, 2586, 5458, 2882, 0, 0, 0, 749, 364, 862, 0, 18, 0, 450, 970, 468, 0, 0, 0, 12181286, 5003992, 3618730, 0, 83101, 0, 418592, 1861909, 5964098, 0, 0, 0, 5362, 2688, 4876, 0, 146, 0, 2476, 5854, 2906 +18,2, 1, 0, 0, 0, 66, 673, 168, 0, 1, 0, 54, 941, 147, 0, 0, 0, 44520, 240880, 38760, 0, 65, 0, 23598, 1225279, 118776, 0, 0, 0, 264, 4280, 843, 0, 6, 0, 294, 7107, 1086, 0, 0, 0, 66, 673, 168, 0, 1, 0, 54, 941, 147, 0, 0, 0, 98352, 284255, 68271, 0, 3800, 0, 59520, 2097265, 304335, 0, 0, 0, 327, 4063, 741, 0, 11, 0, 348, 7381, 1146 +18,2, 2, 0, 0, 0, 166, 168, 128, 0, 3, 0, 251, 605, 278, 0, 0, 0, 102890, 125654, 36266, 0, 5401, 0, 104452, 656539, 236222, 0, 0, 0, 1202, 1314, 816, 0, 26, 0, 1495, 4228, 1914, 0, 0, 0, 166, 168, 128, 0, 3, 0, 251, 605, 278, 0, 0, 0, 2285978, 160172, 2090060, 0, 9814, 0, 300027, 2161250, 373994, 0, 0, 0, 1190, 1262, 870, 0, 31, 0, 1603, 4505, 1958 +17,2, 1, 0, 1, 0, 234, 4631, 213, 0, 3, 0, 159, 2446, 414, 0, 2004, 0, 68274, 2095968, 139872, 0, 1408, 0, 103323, 2442810, 400467, 0, 10, 0, 1140, 33740, 1179, 0, 20, 0, 1026, 17287, 3075, 0, 1, 0, 234, 4631, 213, 0, 3, 0, 159, 2446, 414, 0, 868, 0, 98832, 8702615, 170067, 0, 1542, 0, 218760, 7850635, 808470, 0, 9, 0, 1248, 32277, 1188, 0, 24, 0, 1041, 17956, 2946 +1,2, 2, 0, 0, 0, 37, 6, 14, 0, 0, 0, 2, 30, 2, 0, 0, 0, 4527, 1346, 6416, 0, 0, 0, 24, 16458, 144, 0, 0, 0, 206, 36, 108, 0, 0, 0, 6, 192, 12, 0, 0, 0, 37, 6, 14, 0, 0, 0, 2, 30, 2, 0, 0, 0, 5488, 446, 5250, 0, 0, 0, 30, 10172, 34, 0, 0, 0, 188, 24, 110, 0, 0, 0, 8, 166, 8 +1,2, 1, 0, 0, 0, 9, 24, 6, 0, 2, 0, 18, 134, 30, 0, 0, 0, 675, 7107, 36, 0, 1981, 0, 30018, 135104, 20025, 0, 0, 0, 24, 102, 12, 0, 13, 0, 180, 1029, 222, 0, 0, 0, 9, 24, 6, 0, 2, 0, 18, 134, 30, 0, 0, 0, 636, 2748, 0, 0, 2429, 0, 32481, 4541278, 26700, 0, 0, 0, 24, 90, 0, 0, 17, 0, 183, 1174, 222 +17,2, 3, 0, 0, 0, 4, 35, 1, 3, 14, 0, 122, 251, 29, 0, 0, 0, 1146, 3747, 391, 707, 337, 0, 122661, 121003, 12768, 0, 0, 0, 26, 171, 8, 9, 14, 0, 825, 776, 68, 0, 0, 0, 4, 35, 1, 3, 14, 0, 122, 251, 29, 0, 0, 0, 700, 5339, 340, 3762, 40072, 0, 2832693, 169923, 16034, 0, 0, 0, 20, 216, 8, 20, 99, 0, 827, 1078, 123 +18,2, 3, 0, 0, 0, 19, 32, 2, 11, 17, 0, 275, 285, 16, 0, 0, 0, 13364, 10026, 246, 3114, 280, 0, 357916, 126252, 114149, 0, 0, 0, 133, 205, 12, 31, 26, 0, 2085, 990, 47, 0, 0, 0, 19, 32, 2, 11, 17, 0, 275, 285, 16, 0, 0, 0, 13729, 22024, 133, 12428, 25362, 0, 4092142, 258211, 111210, 0, 0, 0, 128, 215, 11, 81, 115, 0, 2164, 1117, 54 +19,2, 2, 0, 0, 0, 70, 58, 73, 0, 2, 0, 26, 206, 68, 0, 0, 0, 20418, 89810, 27668, 0, 4986, 0, 16416, 198004, 38290, 0, 0, 0, 452, 414, 485, 0, 12, 0, 196, 1454, 416, 0, 0, 0, 70, 58, 73, 0, 2, 0, 26, 206, 68, 0, 0, 0, 63898, 95300, 64398, 0, 2751, 0, 299512, 396977, 31906, 0, 0, 0, 482, 410, 511, 0, 15, 0, 208, 1573, 416 +1,2, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 1008, 0, 0, 0, 0, 0, 12817, 2, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 124, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 3223, 0, 0, 0, 0, 0, 160911, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 161, 0 +18,2, 1, 0, 3, 0, 339, 2196, 291, 0, 32, 0, 303, 4874, 576, 0, 111, 0, 181284, 1159503, 126882, 0, 5010, 0, 409545, 5030984, 829074, 0, 14, 0, 1701, 13924, 1704, 0, 120, 0, 1983, 33746, 4083, 0, 3, 0, 339, 2196, 291, 0, 32, 0, 303, 4874, 576, 0, 3486, 0, 352479, 2674419, 408945, 0, 73833, 0, 760875, 11313206, 1315020, 0, 25, 0, 1938, 13691, 1773, 0, 240, 0, 2142, 36549, 4140 +19,2, 3, 0, 0, 0, 0, 4, 0, 0, 6, 0, 11, 25, 13, 0, 0, 0, 0, 319, 0, 0, 390, 0, 11593, 8531, 254, 0, 0, 0, 0, 23, 0, 0, 36, 0, 88, 101, 39, 0, 0, 0, 0, 4, 0, 0, 6, 0, 11, 25, 13, 0, 0, 0, 0, 411, 0, 0, 22768, 0, 22476, 8088, 5343, 0, 0, 0, 0, 24, 0, 0, 66, 0, 81, 94, 25 +18,2, 2, 0, 0, 0, 704, 506, 688, 0, 53, 0, 930, 1972, 912, 0, 0, 0, 399730, 318736, 286718, 0, 9981, 0, 827452, 1848327, 1126668, 0, 0, 0, 4892, 3620, 4398, 0, 94, 0, 5590, 11785, 5640, 0, 0, 0, 704, 506, 688, 0, 53, 0, 930, 1972, 912, 0, 0, 0, 12741904, 1013578, 5258486, 0, 161877, 0, 1452762, 4523126, 6393588, 0, 0, 0, 4994, 3680, 4564, 0, 347, 0, 5422, 13108, 6054 +1,2, 2, 0, 0, 0, 8, 8, 26, 0, 0, 0, 2, 31, 0, 0, 0, 0, 1094, 6188, 5532, 0, 0, 0, 172, 7042, 0, 0, 0, 0, 30, 66, 122, 0, 0, 0, 12, 162, 0, 0, 0, 0, 8, 8, 26, 0, 0, 0, 2, 31, 0, 0, 0, 0, 1582, 27014, 94360, 0, 0, 0, 1898, 11084, 0, 0, 0, 0, 32, 76, 150, 0, 0, 0, 18, 198, 0 +1,2, 1, 0, 0, 0, 6, 48, 9, 0, 1, 0, 27, 216, 24, 0, 0, 0, 552, 9072, 1335, 0, 65, 0, 24828, 163250, 9225, 0, 0, 0, 24, 111, 57, 0, 6, 0, 192, 1490, 165, 0, 0, 0, 6, 48, 9, 0, 1, 0, 27, 216, 24, 0, 0, 0, 1098, 6177, 2157, 0, 3688, 0, 68982, 291932, 6639, 0, 0, 0, 27, 132, 66, 0, 11, 0, 180, 1566, 165 +19,2, 1, 0, 0, 0, 78, 108, 42, 0, 1, 0, 6, 411, 60, 0, 0, 0, 71706, 42813, 17241, 0, 13, 0, 5739, 310052, 151146, 0, 0, 0, 549, 666, 228, 0, 3, 0, 30, 2892, 504, 0, 0, 0, 78, 108, 42, 0, 1, 0, 6, 411, 60, 0, 0, 0, 82194, 36567, 29760, 0, 111, 0, 3324, 456084, 241281, 0, 0, 0, 597, 588, 288, 0, 6, 0, 48, 2948, 519 +20,2, 2, 0, 0, 0, 120, 96, 149, 0, 1, 0, 68, 347, 216, 0, 0, 0, 53458, 137692, 120982, 0, 15, 0, 43384, 322919, 189918, 0, 0, 0, 870, 760, 1031, 0, 4, 0, 474, 2236, 1514, 0, 0, 0, 120, 96, 149, 0, 1, 0, 68, 347, 216, 0, 0, 0, 130186, 337632, 262366, 0, 237, 0, 42924, 441157, 718354, 0, 0, 0, 920, 768, 1032, 0, 7, 0, 500, 2597, 1666 +1,2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 199, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0 +19,2, 2, 1, 0, 0, 754, 1308, 642, 0, 59, 0, 750, 2503, 1172, 807, 0, 0, 485315, 1553184, 376730, 0, 12186, 0, 1082584, 2445058, 760326, 9, 0, 0, 5413, 10992, 4534, 0, 142, 0, 5472, 15530, 8070, 1, 0, 0, 754, 1308, 642, 0, 59, 0, 750, 2503, 1172, 270, 0, 0, 1355513, 7393388, 1321190, 0, 147565, 0, 2581272, 4853345, 13507294, 8, 0, 0, 5339, 11118, 4800, 0, 408, 0, 5918, 16874, 8904 +19,2, 3, 0, 7, 0, 12, 28, 2, 7, 26, 0, 75, 277, 26, 0, 1363, 0, 5349, 24558, 1199, 296, 2658, 0, 67766, 156552, 237472, 0, 38, 0, 91, 167, 16, 10, 61, 0, 443, 973, 137, 0, 7, 0, 12, 28, 2, 7, 26, 0, 75, 277, 26, 0, 8182, 0, 6687, 27714, 558, 2763, 40502, 0, 84342, 316095, 261864, 0, 38, 0, 92, 185, 13, 48, 181, 0, 491, 1258, 163 +20,2, 3, 0, 0, 0, 0, 11, 8, 0, 9, 0, 4, 47, 8, 0, 0, 0, 0, 3543, 924, 0, 78, 0, 1210, 27600, 469, 0, 0, 0, 0, 70, 46, 0, 11, 0, 23, 198, 8, 0, 0, 0, 0, 11, 8, 0, 9, 0, 4, 47, 8, 0, 0, 0, 0, 6285, 372, 0, 7754, 0, 1594, 47731, 1723, 0, 0, 0, 0, 78, 36, 0, 65, 0, 22, 241, 10 +1,2, 2, 0, 0, 0, 16, 4, 12, 0, 0, 0, 2, 37, 8, 0, 0, 0, 11488, 1402, 3614, 0, 0, 0, 78, 73539, 4104, 0, 0, 0, 100, 28, 76, 0, 0, 0, 10, 281, 32, 0, 0, 0, 16, 4, 12, 0, 0, 0, 2, 37, 8, 0, 0, 0, 60588, 4990, 3220, 0, 0, 0, 468, 69458, 5602, 0, 0, 0, 122, 38, 66, 0, 0, 0, 14, 262, 38 +1,2, 1, 0, 0, 0, 21, 30, 6, 0, 2, 0, 27, 167, 63, 0, 0, 0, 3372, 7836, 867, 0, 150, 0, 21006, 139819, 366273, 0, 0, 0, 57, 111, 42, 0, 8, 0, 213, 1169, 456, 0, 0, 0, 21, 30, 6, 0, 2, 0, 27, 167, 63, 0, 0, 0, 46539, 1728, 366, 0, 1474, 0, 32796, 626064, 534795, 0, 0, 0, 72, 96, 33, 0, 18, 0, 216, 1230, 486 +20,2, 1, 0, 0, 0, 54, 192, 123, 0, 3, 0, 42, 727, 111, 0, 0, 0, 28119, 55029, 53541, 0, 359, 0, 39843, 761323, 191997, 0, 0, 0, 324, 840, 849, 0, 16, 0, 276, 5346, 849, 0, 0, 0, 54, 192, 123, 0, 3, 0, 42, 727, 111, 0, 0, 0, 113976, 141753, 130401, 0, 25885, 0, 44892, 1557215, 267702, 0, 0, 0, 366, 972, 864, 0, 35, 0, 261, 5661, 840 +19,2, 1, 0, 1, 0, 372, 1220, 402, 0, 20, 0, 210, 5855, 654, 0, 29, 0, 177024, 960526, 267018, 0, 10190, 0, 141672, 4883408, 890076, 0, 4, 0, 1974, 7109, 2628, 0, 103, 0, 1353, 40566, 4479, 0, 1, 0, 372, 1220, 402, 0, 20, 0, 210, 5855, 654, 0, 56, 0, 328626, 1536610, 1119162, 0, 48263, 0, 333933, 10965334, 1802859, 0, 5, 0, 2139, 7498, 2754, 0, 164, 0, 1473, 42648, 4869 +20,2, 2, 0, 0, 0, 974, 528, 672, 0, 21, 0, 620, 1945, 857, 0, 0, 0, 755092, 404656, 325964, 0, 377, 0, 1172006, 2453103, 1372977, 0, 0, 0, 7434, 3832, 4516, 0, 24, 0, 4584, 13864, 6069, 0, 0, 0, 974, 528, 672, 0, 21, 0, 620, 1945, 857, 0, 0, 0, 1778036, 1109824, 5048208, 0, 5519, 0, 7805040, 7045231, 4984821, 0, 0, 0, 7586, 4000, 4772, 0, 124, 0, 4770, 14122, 6517 +2,2, 1, 0, 0, 0, 30, 57, 21, 0, 0, 0, 31, 195, 48, 0, 0, 0, 12591, 7011, 2412, 0, 0, 0, 10672, 228399, 29895, 0, 0, 0, 138, 198, 78, 0, 0, 0, 87, 1086, 276, 0, 0, 0, 30, 57, 21, 0, 0, 0, 31, 195, 48, 0, 0, 0, 9624, 9402, 1773, 0, 0, 0, 47350, 312972, 31098, 0, 0, 0, 132, 219, 87, 0, 0, 0, 189, 1242, 300 +21,2, 3, 0, 0, 0, 0, 3, 2, 0, 7, 0, 4, 69, 6, 0, 0, 0, 0, 239, 1101, 0, 138, 0, 6652, 12521, 14292, 0, 0, 0, 0, 17, 15, 0, 15, 0, 22, 170, 45, 0, 0, 0, 0, 3, 2, 0, 7, 0, 4, 69, 6, 0, 0, 0, 0, 283, 4374, 0, 27445, 0, 6750, 22618, 92815, 0, 0, 0, 0, 18, 20, 0, 63, 0, 22, 209, 63 +20,2, 3, 0, 0, 0, 9, 37, 5, 0, 22, 0, 37, 238, 21, 0, 0, 0, 141274, 15186, 1244, 0, 16877, 0, 136458, 129402, 49477, 0, 0, 0, 76, 248, 32, 0, 43, 0, 351, 849, 172, 0, 0, 0, 9, 37, 5, 0, 22, 0, 37, 238, 21, 0, 0, 0, 144108, 53644, 2220, 0, 78310, 0, 147825, 233370, 88103, 0, 0, 0, 78, 269, 28, 0, 151, 0, 356, 1074, 186 +2,2, 2, 0, 0, 0, 30, 10, 26, 0, 0, 0, 10, 68, 61, 0, 0, 0, 85264, 3164, 23244, 0, 0, 0, 4840, 156100, 245982, 0, 0, 0, 220, 74, 198, 0, 0, 0, 72, 514, 475, 0, 0, 0, 30, 10, 26, 0, 0, 0, 10, 68, 61, 0, 0, 0, 89482, 2042, 26556, 0, 0, 0, 21770, 168140, 276040, 0, 0, 0, 188, 68, 194, 0, 0, 0, 80, 530, 501 +21,2, 2, 0, 0, 0, 254, 102, 290, 0, 3, 0, 100, 276, 142, 0, 0, 0, 101408, 40150, 136656, 0, 2, 0, 85200, 403658, 72700, 0, 0, 0, 1820, 696, 2024, 0, 1, 0, 674, 1939, 900, 0, 0, 0, 254, 102, 290, 0, 3, 0, 100, 276, 142, 0, 0, 0, 256220, 94720, 218312, 0, 1218, 0, 318724, 546249, 116866, 0, 0, 0, 1848, 676, 2020, 0, 20, 0, 792, 2077, 974 +20,2, 1, 0, 2, 0, 297, 1291, 546, 0, 20, 0, 189, 3938, 723, 0, 0, 0, 109734, 557607, 253392, 0, 1390, 0, 99123, 3771038, 1200825, 0, 0, 0, 1611, 7833, 3546, 0, 76, 0, 1143, 27920, 5415, 0, 2, 0, 297, 1291, 546, 0, 20, 0, 189, 3938, 723, 0, 335, 0, 366291, 1342711, 2032848, 0, 33962, 0, 150069, 8369832, 2209548, 0, 14, 0, 1749, 7987, 3966, 0, 162, 0, 1254, 29849, 5670 +21,2, 1, 0, 0, 0, 48, 325, 84, 0, 3, 0, 78, 981, 180, 0, 0, 0, 9642, 363088, 32388, 0, 1344, 0, 31680, 836172, 298425, 0, 0, 0, 168, 1972, 420, 0, 19, 0, 369, 7085, 1341, 0, 0, 0, 48, 325, 84, 0, 3, 0, 78, 981, 180, 0, 0, 0, 12465, 502591, 53706, 0, 7645, 0, 181251, 1713008, 517632, 0, 0, 0, 222, 1882, 513, 0, 31, 0, 513, 7457, 1314 +2,2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8454, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110009, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 6 +2,2, 1, 0, 0, 0, 123, 462, 87, 0, 4, 0, 219, 1389, 429, 0, 0, 0, 68547, 164361, 56235, 0, 577, 0, 223839, 2046809, 466341, 0, 0, 0, 534, 2532, 465, 0, 27, 0, 1479, 10402, 3177, 0, 0, 0, 123, 462, 87, 0, 4, 0, 219, 1389, 429, 0, 0, 0, 112542, 5979582, 229926, 0, 5527, 0, 499440, 4514245, 1215558, 0, 0, 0, 540, 2523, 525, 0, 34, 0, 1683, 10855, 3279 +21,2, 3, 0, 1, 0, 3, 55, 7, 0, 26, 0, 14, 232, 31, 0, 128, 0, 3297, 35505, 3338, 0, 948, 0, 29020, 94107, 28139, 0, 7, 0, 23, 380, 51, 0, 42, 0, 101, 922, 220, 0, 1, 0, 3, 55, 7, 0, 26, 0, 14, 232, 31, 0, 25, 0, 55991, 53761, 3095, 0, 9536, 0, 47884, 238374, 171680, 0, 4, 0, 33, 405, 52, 0, 165, 0, 108, 1068, 246 +2,2, 2, 0, 0, 0, 1497, 140, 186, 0, 0, 0, 98, 598, 188, 0, 0, 0, 1181685, 99136, 115096, 0, 0, 0, 206424, 593736, 465832, 0, 0, 0, 11827, 1020, 1244, 0, 0, 0, 666, 3446, 1438, 0, 0, 0, 1497, 140, 186, 0, 0, 0, 98, 598, 188, 0, 0, 0, 3103086, 387794, 632904, 0, 0, 0, 1015878, 799352, 1829496, 0, 0, 0, 11263, 1082, 1296, 0, 0, 0, 604, 3844, 1570 +22,2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 282, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34645, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15 +21,2, 2, 0, 0, 0, 706, 502, 820, 0, 17, 0, 552, 1512, 860, 0, 0, 0, 398350, 294482, 515398, 0, 3326, 0, 332358, 2981024, 992538, 0, 0, 0, 5086, 3402, 5308, 0, 56, 0, 2968, 10205, 5508, 0, 0, 0, 706, 502, 820, 0, 17, 0, 552, 1512, 860, 0, 0, 0, 1774456, 458106, 1443202, 0, 82151, 0, 630886, 4642342, 2815206, 0, 0, 0, 5182, 3422, 5408, 0, 133, 0, 2930, 10777, 5904 +2,2, 3, 0, 0, 0, 1, 29, 1, 0, 1, 0, 6, 237, 9, 0, 0, 0, 1188, 10039, 8, 0, 65, 0, 98, 90236, 2284, 0, 0, 0, 10, 179, 3, 0, 6, 0, 9, 1294, 27, 0, 0, 0, 1, 29, 1, 0, 1, 0, 6, 237, 9, 0, 0, 0, 1559, 14914, 6, 0, 3682, 0, 794, 245376, 1369, 0, 0, 0, 10, 203, 2, 0, 11, 0, 13, 1338, 32 +21,2, 1, 0, 0, 0, 228, 933, 351, 0, 21, 0, 219, 3412, 573, 0, 0, 0, 180690, 652527, 391029, 0, 2539, 0, 94782, 3752735, 835020, 0, 0, 0, 1197, 5649, 2031, 0, 79, 0, 1272, 25068, -2147479490, 0, 0, 0, 228, 933, 351, 0, 21, 0, 219, 3412, 573, 0, 0, 0, 184323, 987831, 501957, 0, 15118, 0, 212391, 7449421, 1749906, 0, 0, 0, 1287, 5850, 2091, 0, 151, 0, 1425, 26144, -2147479229 +16,1, 2, 0, 0, 0, 379, 308, 472, 0, 42, 0, 716, 1294, 630, 0, 0, 0, 321120, 560954, 468796, 0, 5090, 0, 1169452, 1853062, 1047122, 0, 0, 0, 2589, 2598, 3458, 0, 107, 0, 5700, 9189, 4588, 0, 0, 0, 379, 308, 472, 0, 42, 0, 716, 1294, 630, 0, 0, 0, 631404, 773810, 900690, 0, 64123, 0, 2765048, 8700173, 5158512, 0, 0, 0, 2601, 2562, 3394, 0, 306, 0, 6012, 10442, 5166 +16,1, 1, 0, 2, 0, 219, 751, 171, 0, 8, 0, 117, 2129, 378, 0, 455, 0, 67023, 730162, 131841, 0, 704, 0, 141645, 2683057, 634212, 0, 14, 0, 939, 3847, 1266, 0, 16, 0, 831, 15926, 2937, 0, 2, 0, 219, 751, 171, 0, 8, 0, 117, 2129, 378, 0, 388, 0, 69867, 1192088, 177882, 0, 4138, 0, 200115, 4332990, 1109751, 0, 14, 0, 963, 4015, 1245, 0, 57, 0, 819, 16545, 3009 +15,1, 1, 0, 0, 0, 255, 2400, 240, 0, 18, 0, 327, 2791, 432, 0, 0, 0, 82095, 1115169, 129222, 0, 6895, 0, 410172, 3194342, 1145988, 0, 0, 0, 981, 16071, 1578, 0, 51, 0, 2589, 20356, 3492, 0, 0, 0, 255, 2400, 240, 0, 18, 0, 327, 2791, 432, 0, 0, 0, 127872, 5910249, 203661, 0, 88978, 0, 794538, 6765101, 1919268, 0, 0, 0, 1083, 15231, 1626, 0, 150, 0, 2862, 21579, 3639 +15,1, 3, 0, 0, 0, 2, 40, 1, 0, 8, 0, 6, 79, 16, 0, 0, 0, 2121, 209801, 97, 0, 422, 0, 7098, 29306, 136782, 0, 0, 0, 16, 285, 6, 0, 17, 0, 46, 378, 112, 0, 0, 0, 2, 40, 1, 0, 8, 0, 6, 79, 16, 0, 0, 0, 1865, 253154, 21, 0, 53756, 0, 6606, 68467, 252337, 0, 0, 0, 16, 274, 4, 0, 61, 0, 45, 456, 133 +16,1, 3, 0, 1, 0, 0, 16, 5, 0, 7, 0, 3, 62, 2, 0, 15378, 0, 0, 6368, 5282, 0, 2570, 0, 1227, 19554, 187, 0, 13, 0, 0, 111, 45, 0, 38, 0, 21, 246, 7, 0, 1, 0, 0, 16, 5, 0, 7, 0, 3, 62, 2, 0, 15308, 0, 0, 5010, 6847, 0, 8394, 0, 722, 25859, 148, 0, 13, 0, 0, 113, 45, 0, 59, 0, 18, 350, 8 +15,1, 2, 0, 3, 0, 760, 531, 550, 0, 76, 0, 836, 2234, 1045, 0, 4507, 0, 821084, 615873, 423980, 0, 24593, 0, 1300056, 2623061, 1160386, 0, 30, 0, 5486, 4300, 3716, 0, 271, 0, 6466, 15269, 7794, 0, 3, 0, 760, 531, 550, 0, 76, 0, 836, 2234, 1045, 0, 2177, 0, 4265928, 1397461, 1158506, 0, 2011795, 0, 3502758, 8408463, 5870061, 0, 27, 0, 5592, 4343, 3898, 0, 606, 0, 6472, 18102, 8692 +2,2, 1, 0, 0, 0, 39, 141, 78, 0, 0, 0, 87, 400, 132, 0, 0, 0, 17919, 98433, 102642, 0, 0, 0, 66279, 670440, 173421, 0, 0, 0, 240, 756, 540, 0, 0, 0, 510, 2686, 954, 0, 0, 0, 39, 141, 78, 0, 0, 0, 87, 400, 132, 0, 0, 0, 42723, 212679, 151311, 0, 0, 0, 75351, 785697, 325341, 0, 0, 0, 264, 834, 507, 0, 0, 0, 591, 2974, 975 +3,2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0 +22,2, 3, 0, 0, 0, 4, 30, 6, 0, 2, 0, 5, 225, 13, 0, 0, 0, 1105, 1637, 3944, 0, 15, 0, 1067, 65506, 2980, 0, 0, 0, 29, 152, 45, 0, 4, 0, 18, 807, 24, 0, 0, 0, 4, 30, 6, 0, 2, 0, 5, 225, 13, 0, 0, 0, 975, 3008, 3446, 0, 648, 0, 1018, 132744, 2801, 0, 0, 0, 25, 176, 41, 0, 13, 0, 20, 889, 30 +2,2, 2, 0, 0, 0, 204, 72, 66, 0, 8, 0, 90, 233, 54, 0, 0, 0, 156908, 66388, 70482, 0, 7, 0, 30560, 311040, 41386, 0, 0, 0, 1552, 532, 520, 0, 3, 0, 378, 1461, 352, 0, 0, 0, 204, 72, 66, 0, 8, 0, 90, 233, 54, 0, 0, 0, 166986, 86212, 444052, 0, 20145, 0, 49976, 498938, 39486, 0, 0, 0, 1466, 522, 536, 0, 57, 0, 212, 1621, 308 +22,2, 2, 0, 0, 0, 488, 260, 496, 0, 11, 0, 852, 1392, 796, 0, 0, 0, 245628, 191264, 347380, 0, 3555, 0, 433722, 2769790, 392970, 0, 0, 0, 3422, 1954, 3310, 0, 33, 0, 3368, 8740, 3522, 0, 0, 0, 488, 260, 496, 0, 11, 0, 852, 1392, 796, 0, 0, 0, 605302, 241658, 475138, 0, 57908, 0, 3253134, 5715484, 701806, 0, 0, 0, 3504, 1924, 3350, 0, 94, 0, 2970, 8826, 3454 +3,2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0 +22,2, 1, 0, 0, 0, 228, 2376, 277, 0, 10, 0, 156, 2228, 231, 0, 0, 0, 72318, 808374, 225502, 0, 3722, 0, 102804, 2155828, 184323, 0, 0, 0, 1068, 15645, 1540, 0, 59, 0, 663, 15241, 1641, 0, 0, 0, 228, 2376, 277, 0, 10, 0, 156, 2228, 231, 0, 0, 0, 136713, 1609452, 742593, 0, 5807, 0, 90687, 5957734, 291312, 0, 0, 0, 1170, 14916, 1820, 0, 76, 0, 828, 15839, 1755 +2,2, 3, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 47, 13, 0, 0, 0, 0, 175, 0, 0, 0, 0, 2443, 9022, 1416, 0, 0, 0, 0, 25, 0, 0, 0, 0, 11, 122, 35, 0, 0, 0, 0, 5, 0, 0, 0, 0, 1, 47, 13, 0, 0, 0, 0, 450, 0, 0, 0, 0, 2806, 339875, 971, 0, 0, 0, 0, 30, 0, 0, 0, 0, 11, 159, 35 +24,2, 2, 0, 0, 0, 118, 120, 116, 0, 5, 0, 178, 582, 734, 0, 0, 0, 47742, 97392, 49698, 0, 335, 0, 184984, 869918, 589048, 0, 0, 0, 776, 970, 796, 0, 9, 0, 1412, 3829, 5220, 0, 0, 0, 118, 120, 116, 0, 5, 0, 178, 582, 734, 0, 0, 0, 84564, 303924, 468666, 0, 1413, 0, 475828, 1559455, 2934108, 0, 0, 0, 802, 1016, 920, 0, 31, 0, 1464, 4456, 5694 +23,2, 3, 0, 1, 0, 3, 29, 3, 0, 3, 0, 18, 228, 15, 0, 253, 0, 1645, 6587, 5948, 0, 132, 0, 10198, 111231, 116122, 0, 7, 0, 19, 171, 21, 0, 13, 0, 108, 615, 67, 0, 1, 0, 3, 29, 3, 0, 3, 0, 18, 228, 15, 0, 816, 0, 826, 6717, 5845, 0, 8629, 0, 21288, 175361, 112107, 0, 9, 0, 21, 187, 19, 0, 31, 0, 118, 700, 79 +24,2, 3, 0, 0, 0, 0, 13, 0, 0, 1, 0, 5, 147, 7, 0, 0, 0, 0, 1796, 0, 0, 65, 0, 2260, 128802, 1247, 0, 0, 0, 0, 68, 0, 0, 6, 0, 26, 903, 49, 0, 0, 0, 0, 13, 0, 0, 1, 0, 5, 147, 7, 0, 0, 0, 0, 2801, 0, 0, 3818, 0, 1187, 300393, 955, 0, 0, 0, 0, 87, 0, 0, 11, 0, 35, 1020, 45 +2,2, 1, 0, 0, 0, 36, 51, 33, 0, 0, 0, 18, 148, 24, 0, 0, 0, 6987, 14220, 29367, 0, 0, 0, 2871, 1237496, 14181, 0, 0, 0, 120, 231, 117, 0, 0, 0, 72, 1167, 147, 0, 0, 0, 36, 51, 33, 0, 0, 0, 18, 148, 24, 0, 0, 0, 8757, 26379, 25293, 0, 0, 0, 3264, 1636930, 32115, 0, 0, 0, 141, 243, 141, 0, 0, 0, 105, 1191, 180 +2,2, 2, 0, 0, 0, 42, 32, 30, 0, 0, 0, 14, 54, 31, 0, 0, 0, 31926, 13508, 7102, 0, 0, 0, 4610, 42054, 147876, 0, 0, 0, 334, 204, 192, 0, 0, 0, 66, 382, 246, 0, 0, 0, 42, 32, 30, 0, 0, 0, 14, 54, 31, 0, 0, 0, 134636, 13632, 6648, 0, 0, 0, 4798, 59930, 166658, 0, 0, 0, 350, 210, 178, 0, 0, 0, 92, 382, 266 +22,2, 3, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 55, 3, 0, 0, 0, 0, 871, 0, 0, 0, 0, 30, 16373, 1995, 0, 0, 0, 0, 44, 0, 0, 0, 0, 6, 136, 10, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 55, 3, 0, 0, 0, 0, 770, 0, 0, 0, 0, 16089, 43624, 1592, 0, 0, 0, 0, 41, 0, 0, 0, 0, 22, 194, 11 +23,2, 1, 0, 0, 0, 321, 1545, 303, 0, 14, 0, 153, 1964, 255, 0, 0, 0, 210276, 1182864, 205137, 0, 2573, 0, 93216, 1841691, 296244, 0, 0, 0, 1383, -2147473874, 1686, 0, 45, 0, 798, 13227, 1701, 0, 0, 0, 321, 1545, 303, 0, 14, 0, 153, 1964, 255, 0, 0, 0, 305676, 1901511, 563577, 0, 47653, 0, 143316, 2747785, 1018167, 0, 0, 0, 1527, -2147473778, 1902, 0, 121, 0, 1002, 13916, 1806 +23,2, 2, 0, 0, 0, 650, 474, 553, 0, 15, 0, 378, 2475, 1150, 0, 0, 0, 351492, 505166, 356013, 0, 3015, 0, 516980, 4199097, 1077868, 0, 0, 0, 4562, 3452, 3684, 0, 42, 0, 2132, 16506, 7270, 0, 0, 0, 650, 474, 553, 0, 15, 0, 378, 2475, 1150, 0, 0, 0, 2680958, 984610, 650208, 0, 67923, 0, 1107702, 8273365, 2680390, 0, 0, 0, 4512, 3628, 3724, 0, 129, 0, 2410, 18175, 8124 +22,2, 2, 0, 0, 0, 90, 28, 44, 0, 5, 0, 73, 171, 92, 0, 0, 0, 28138, 3860, 13378, 0, 232, 0, 36640, 341130, 38196, 0, 0, 0, 632, 160, 300, 0, 9, 0, 470, 1071, 418, 0, 0, 0, 90, 28, 44, 0, 5, 0, 73, 171, 92, 0, 0, 0, 35334, 1902, 56688, 0, 2271, 0, 111140, 654349, 51360, 0, 0, 0, 630, 136, 336, 0, 34, 0, 454, 1124, 498 +22,2, 1, 0, 0, 0, 48, 2440, 33, 0, 4, 0, 27, 218, 42, 0, 0, 0, 27159, 996768, 18327, 0, 1986, 0, 18153, 126693, 40923, 0, 0, 0, 276, 18271, 228, 0, 15, 0, 126, 1404, 315, 0, 0, 0, 48, 2440, 33, 0, 4, 0, 27, 218, 42, 0, 0, 0, 16662, 3809734, 17001, 0, 3604, 0, 8451, 207002, 46002, 0, 0, 0, 282, 17442, 237, 0, 33, 0, 162, 1512, 324 +2,2, 3, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 117, 357, 0, 0, 0, 0, 0, 10414, 0, 0, 0, 0, 6, 28, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 164, 476, 0, 0, 0, 0, 0, 13833, 0, 0, 0, 0, 7, 29, 0, 0, 0, 0, 0, 106, 0 +24,2, 1, 0, 0, 0, 48, 285, 100, 0, 1, 0, 45, 815, 108, 0, 0, 0, 26256, 211101, 94022, 0, 1969, 0, 20361, 709766, 50607, 0, 0, 0, 309, 1884, 669, 0, 10, 0, 210, 5447, 720, 0, 0, 0, 48, 285, 100, 0, 1, 0, 45, 815, 108, 0, 0, 0, 73866, 368856, 177028, 0, 2357, 0, 159048, 1223311, 81957, 0, 0, 0, 327, 1971, 663, 0, 11, 0, 324, 6049, 750 +24,2, 2, 0, 0, 0, 1032, 540, 1028, 0, 29, 0, 486, 2591, 911, 0, 0, 0, 707574, 342194, 523848, 0, 65088, 0, 526036, 3497042, 1070710, 0, 0, 0, 7686, 4108, 7406, 0, 116, 0, 3586, 17926, 6619, 0, 0, 0, 1032, 540, 1028, 0, 29, 0, 486, 2591, 911, 0, 0, 0, 4185208, 4971920, 6283804, 0, 321590, 0, 1216890, 14197484, 8077992, 0, 0, 0, 8008, 4238, 7482, 0, 237, 0, 3964, 19825, 7154 +24,2, 3, 0, 0, 0, 9, 20, 1, 1, 37, 0, 42, 244, 13, 0, 0, 0, 4830, 4795, 33, 359, 7975, 0, 38773, 241551, 119736, 0, 0, 0, 59, 127, 5, 8, 103, 0, 281, 1223, 103, 0, 0, 0, 9, 20, 1, 1, 37, 0, 42, 244, 13, 0, 0, 0, 3174, 6695, 16, 2433, 76753, 0, 57079, 343279, 124293, 0, 0, 0, 63, 134, 4, 11, 267, 0, 310, 1459, 118 +25,2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0 +25,2, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 1260, 0, 0, 0, 0, 0, 25781, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 133, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 2424, 0, 0, 0, 0, 0, 22428, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 132, 0 +24,2, 1, 0, 7, 0, 1041, 3345, 1062, 0, 76, 0, 438, 10532, 951, 0, 74, 0, 1064565, 3373064, 884409, 0, 16161, 0, 556020, 10590912, 1016544, 0, 6, 0, 7944, 24699, 8052, 0, 229, 0, 3333, 76541, 6957, 0, 7, 0, 1041, 3345, 1062, 0, 76, 0, 438, 10532, 951, 0, 880, 0, 2848071, 8603619, 2260227, 0, 70574, 0, 1018164, 29913706, 3666252, 0, 35, 0, 8406, 25685, 8466, 0, 495, 0, 3456, 82866, 7710 +24,2, 2, 0, 0, 0, 138, 141, 202, 0, 9, 0, 134, 995, 446, 0, 0, 0, 74726, 94968, 138794, 0, 226, 0, 155802, 1791298, 1080908, 0, 0, 0, 958, 1142, 1486, 0, 28, 0, 1008, 6788, 3414, 0, 0, 0, 138, 141, 202, 0, 9, 0, 134, 995, 446, 0, 0, 0, 1429492, 469718, 521610, 0, 8600, 0, 8482942, 3143894, 1392084, 0, 0, 0, 1078, 1257, 1584, 0, 64, 0, 1088, 7712, 3684 +2,2, 1, 0, 0, 0, 15, 21, 6, 0, 0, 0, 33, 67, 18, 0, 0, 0, 4971, 2574, 3648, 0, 0, 0, 33885, 100276, 3861, 0, 0, 0, 99, 69, 54, 0, 0, 0, 135, 414, 75, 0, 0, 0, 15, 21, 6, 0, 0, 0, 33, 67, 18, 0, 0, 0, 2970, 176142, 5010, 0, 0, 0, 42738, 126186, 2079, 0, 0, 0, 84, 120, 54, 0, 0, 0, 195, 414, 60 +24,2, 3, 0, 0, 0, 1, 7, 0, 0, 0, 0, 5, 43, 2, 0, 0, 0, 60, 3702, 0, 0, 0, 0, 9549, 9636, 25, 0, 0, 0, 5, 49, 0, 0, 0, 0, 50, 85, 4, 0, 0, 0, 1, 7, 0, 0, 0, 0, 5, 43, 2, 0, 0, 0, 103, 5261, 0, 0, 0, 0, 7278, 46174, 176, 0, 0, 0, 6, 54, 0, 0, 0, 0, 45, 136, 8 +25,2, 2, 0, 0, 0, 370, 344, 336, 0, 19, 0, 292, 920, 324, 0, 0, 0, 256928, 270820, 185588, 0, 117, 0, 289690, 1151465, 385376, 0, 0, 0, 2680, 2696, 2344, 0, 11, 0, 2286, 6135, 2302, 0, 0, 0, 370, 344, 336, 0, 19, 0, 292, 920, 324, 0, 0, 0, 3829852, 1212070, 1026366, 0, 4471, 0, 630464, 2882280, 711710, 0, 0, 0, 2858, 2974, 2472, 0, 101, 0, 2312, 6969, 2484 +2,2, 2, 0, 0, 0, 32, 6, 14, 0, 0, 0, 6, 36, 13, 0, 0, 0, 17240, 860, 7518, 0, 0, 0, 38866, 26280, 12784, 0, 0, 0, 242, 34, 110, 0, 0, 0, 66, 246, 117, 0, 0, 0, 32, 6, 14, 0, 0, 0, 6, 36, 13, 0, 0, 0, 58820, 2494, 21600, 0, 0, 0, 41212, 27294, 11102, 0, 0, 0, 260, 40, 126, 0, 0, 0, 62, 272, 112 +25,2, 3, 0, 4, 0, 2, 11, 1, 1, 20, 0, 0, 176, 3, 0, 0, 0, 1684, 2280, 1641, 1662, 2839, 0, 0, 159278, 4077, 0, 0, 0, 14, 67, 10, 10, 60, 0, 0, 613, 22, 0, 4, 0, 2, 11, 1, 1, 20, 0, 0, 176, 3, 0, 434, 0, 3303, 1957, 827, 10804, 51598, 0, 0, 223620, 3438, 0, 18, 0, 14, 71, 9, 13, 158, 0, 0, 816, 21 +25,2, 1, 0, 2, 0, 279, 1162, 247, 1, 32, 0, 182, 4063, 876, 0, 513, 0, 170148, 1273974, 133553, 6, 3302, 0, 192447, 4054780, 708768, 0, 9, 0, 1698, 8304, 1737, 2, 77, 0, 1099, 29107, 4737, 0, 2, 0, 279, 1162, 247, 1, 32, 0, 182, 4063, 876, 0, 340, 0, 291144, 2393639, 199939, 18423, 96365, 0, 273435, 8167087, 6460545, 0, 14, 0, 1704, 8551, 1795, 14, 228, 0, 1387, 31419, 4866 +2,2, 3, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 8596, 0, 0, 0, 0, 0, 997, 3713, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 44, 11, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 10173, 0, 0, 0, 0, 0, 2197, 3603, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 51, 11 +24,2, 1, 0, 0, 0, 123, 483, 94, 0, 9, 0, 51, 897, 414, 0, 0, 0, 81924, 225951, 45107, 0, 48, 0, 35499, 591669, 234513, 0, 0, 0, 741, 3108, 594, 0, 14, 0, 348, -2147477656, 2706, 0, 0, 0, 123, 483, 94, 0, 9, 0, 51, 897, 414, 0, 0, 0, 98988, 625935, 157871, 0, 37899, 0, 149508, 1103013, 784575, 0, 0, 0, 762, 3156, 690, 0, 68, 0, 402, -2147477443, 3135 +25,2, 2, 0, 2, 0, 309, 152, 254, 0, 6, 0, 146, 856, 216, 0, 3045, 0, 163029, 86841, 164366, 0, 8, 0, 161328, 1530376, 195580, 0, 18, 0, 2241, 1142, 1722, 0, 3, 0, 1026, 6083, 1494, 0, 2, 0, 309, 152, 254, 0, 6, 0, 146, 856, 216, 0, 7058, 0, 2450659, 353160, 574918, 0, 1747, 0, 750760, 3603459, 419102, 0, 17, 0, 2271, 1221, 1812, 0, 38, 0, 1138, 6498, 1766 +25,2, 3, 0, 0, 0, 2, 13, 0, 0, 12, 0, 9, 92, 18, 0, 0, 0, 953, 7946, 0, 0, 212, 0, 12773, 46474, 9785, 0, 0, 0, 15, 89, 0, 0, 19, 0, 64, 399, 75, 0, 0, 0, 2, 13, 0, 0, 12, 0, 9, 92, 18, 0, 0, 0, 295, 4803, 0, 0, 59442, 0, 55896, 57713, 9352, 0, 0, 0, 13, 89, 0, 0, 93, 0, 65, 474, 67 +25,2, 1, 0, 0, 0, 306, 1161, 198, 0, 16, 0, 111, 2736, 357, 0, 0, 0, 194637, 862065, 220671, 0, 4253, 0, 64446, 3733081, 494295, 0, 0, 0, 2106, 8598, 1449, 0, 70, 0, 690, 19953, 2412, 0, 0, 0, 306, 1161, 198, 0, 16, 0, 111, 2736, 357, 0, 0, 0, 553509, 1970613, 329220, 0, 6594, 0, 128211, 13148501, 804165, 0, 0, 0, 2265, 8727, 1428, 0, 115, 0, 822, 21246, 2367 +26,2, 2, 0, 1, 0, 128, 121, 181, 0, 6, 0, 32, 214, 90, 0, 74, 0, 50474, 88326, 91409, 0, 9, 0, 124894, 218531, 55864, 0, 6, 0, 814, 928, 863, 0, 4, 0, 246, 1346, 668, 0, 1, 0, 128, 121, 181, 0, 6, 0, 32, 214, 90, 0, 222, 0, 137402, 376544, 223010, 0, 1917, 0, 123964, 476877, 94028, 0, 7, 0, 852, 993, 1218, 0, 41, 0, 236, 1583, 702 +2,2, 1, 0, 0, 0, 3, 30, 9, 0, 0, 0, 27, 91, 33, 0, 0, 0, 3, 4515, 1749, 0, 0, 0, 2199, 64706, 1971, 0, 0, 0, 3, 147, 66, 0, 0, 0, 120, 585, 153, 0, 0, 0, 3, 30, 9, 0, 0, 0, 27, 91, 33, 0, 0, 0, 6, 5559, 1134, 0, 0, 0, 12438, 75388, 3675, 0, 0, 0, 3, 138, 57, 0, 0, 0, 165, 700, 177 +26,2, 1, 0, 0, 0, 84, 312, 105, 0, 12, 0, 54, 1018, 90, 0, 0, 0, 38142, 191931, 65286, 0, 981, 0, 22167, 848657, 157629, 0, 0, 0, 471, 2121, 639, 0, 37, 0, 297, 7354, 642, 0, 0, 0, 84, 312, 105, 0, 12, 0, 54, 1018, 90, 0, 0, 0, 108501, 430836, 166179, 0, 11168, 0, 23145, 1759046, 311493, 0, 0, 0, 504, 2190, 690, 0, 98, 0, 342, 7915, 675 +25,2, 2, 0, 1, 0, 420, 239, 697, 0, 19, 0, 230, 1001, 358, 0, 10, 0, 288468, 164118, 275729, 0, 3661, 0, 218080, 1245305, 429584, 0, 3, 0, 2896, 1771, 3716, 0, 48, 0, 1742, 6970, 2478, 0, 1, 0, 420, 239, 697, 0, 19, 0, 230, 1001, 358, 0, 2, 0, 5215282, 675262, 5089850, 0, 62852, 0, 900756, 8198526, 864932, 0, 1, 0, 3046, 1875, 4702, 0, 138, 0, 1840, 7642, 2658 +26,2, 3, 0, 0, 0, 5, 2, 0, 0, 8, 0, 2, 76, 0, 0, 0, 0, 2090, 95, 0, 0, 134, 0, 0, 53228, 0, 0, 0, 0, 28, 10, 0, 0, 14, 0, 0, 400, 0, 0, 0, 0, 5, 2, 0, 0, 8, 0, 2, 76, 0, 0, 0, 0, 1981, 106, 0, 0, 10173, 0, 0, 99473, 0, 0, 0, 0, 39, 9, 0, 0, 64, 0, 0, 470, 0 +2,2, 2, 0, 0, 0, 30, 6, 22, 0, 0, 0, 1, 24, 18, 0, 0, 0, 13148, 4406, 13746, 0, 0, 0, 4, 12294, 147444, 0, 0, 0, 220, 44, 172, 0, 0, 0, 2, 144, 158, 0, 0, 0, 30, 6, 22, 0, 0, 0, 1, 24, 18, 0, 0, 0, 13976, 6996, 11200, 0, 0, 0, 5, 36274, 167384, 0, 0, 0, 210, 48, 172, 0, 0, 0, 2, 182, 166 +25,2, 3, 0, 0, 0, 0, 11, 0, 0, 8, 0, 12, 138, 7, 0, 0, 0, 0, 3685, 0, 0, 987, 0, 1875, 122945, 3170, 0, 0, 0, 0, 70, 0, 0, 38, 0, 17, 421, 21, 0, 0, 0, 0, 11, 0, 0, 8, 0, 12, 138, 7, 0, 0, 0, 0, 4701, 0, 0, 16993, 0, 1832, 136531, 3350, 0, 0, 0, 0, 73, 0, 0, 71, 0, 20, 569, 21 +25,2, 1, 0, 0, 0, 261, 780, 268, 0, 29, 0, 99, 3493, 348, 0, 0, 0, 138798, 510159, 279493, 0, 13406, 0, 111240, 4328326, 770370, 0, 0, 0, 1494, 5106, 1900, 0, 136, 0, 747, 25406, 2703, 0, 0, 0, 261, 780, 268, 0, 29, 0, 99, 3493, 348, 0, 0, 0, 291639, 1683123, 621325, 0, 79177, 0, 147654, 8006834, 1578876, 0, 0, 0, 1593, 5202, 1996, 0, 273, 0, 789, 26637, 2778 +2,2, 3, 0, 0, 0, 1, 0, 0, 0, 3, 0, 2, 7, 0, 0, 0, 0, 927, 0, 0, 0, 1, 0, 0, 728, 0, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 2, 7, 0, 0, 0, 0, 1043, 0, 0, 0, 1131, 0, 0, 6814, 0, 0, 0, 0, 10, 0, 0, 0, 19, 0, 0, 22, 0 +26,2, 2, 0, 2, 0, 1078, 576, 632, 0, 27, 0, 372, 2172, 602, 0, 20, 0, 613196, 533604, 593480, 0, 2657, 0, 409416, 2142931, 1184926, 0, 6, 0, 7162, 4356, 4200, 0, 71, 0, 2710, 14727, 4130, 0, 2, 0, 1078, 576, 632, 0, 27, 0, 372, 2172, 602, 0, 14, 0, 2386514, 1375460, 1558270, 0, 31857, 0, 1156366, 33465108, 1504692, 0, 6, 0, 7470, 4460, 4460, 0, 188, 0, 2916, 16115, 4442 +27,2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 16, 0 +26,2, 1, 0, 0, 0, 510, 2952, 561, 0, 34, 0, 447, 8199, 678, 0, 0, 0, 294282, 2699493, 323127, 0, 14426, 0, 357321, 8295903, 925560, 0, 0, 0, 3144, 21402, 3759, 0, 147, 0, 3147, 59501, 5037, 0, 0, 0, 510, 2952, 561, 0, 34, 0, 447, 8199, 678, 0, 0, 0, 696357, 7048908, 636210, 0, 56545, 0, 973041, 25388082, 2455161, 0, 0, 0, 3186, 21807, 3894, 0, 279, 0, 3579, 63677, 5169 +27,2, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 772, 0, 0, 0, 0, 26735, 22190, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 80, 26, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 7, 2, 0, 0, 0, 0, 0, 884, 0, 0, 0, 0, 26253, 22238, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 74, 26 +26,2, 3, 1, 0, 0, 15, 33, 4, 0, 38, 0, 41, 207, 9, 814, 0, 0, 13467, 8526, 823, 0, 37420, 0, 48624, 224640, 231092, 9, 0, 0, 90, 210, 22, 0, 107, 0, 253, 1070, 87, 1, 0, 0, 15, 33, 4, 0, 38, 0, 41, 207, 9, 144, 0, 0, 39525, 66362, 728, 0, 129731, 0, 126456, 503436, 226872, 7, 0, 0, 116, 234, 22, 0, 304, 0, 299, 1284, 91 +27,2, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 4227, 0, 0, 0, 0, 0, 5142, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 1647, 0, 0, 0, 0, 0, 2571, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 105, 0 +26,2, 2, 0, 0, 0, 344, 78, 116, 0, 3, 0, 58, 188, 84, 0, 0, 0, 158142, 49214, 60658, 0, 599, 0, 95194, 136770, 75446, 0, 0, 0, 2506, 614, 832, 0, 15, 0, 490, 1253, 528, 0, 0, 0, 344, 78, 116, 0, 3, 0, 58, 188, 84, 0, 0, 0, 534774, 383038, 129736, 0, 1388, 0, 99470, 305103, 132310, 0, 0, 0, 2418, 670, 858, 0, 25, 0, 512, 1295, 600 +27,2, 3, 0, 0, 0, 1, 7, 0, 0, 3, 0, 6, 71, 2, 0, 0, 0, 198, 321, 0, 0, 2, 0, 4623, 140674, 1015, 0, 0, 0, 7, 32, 0, 0, 1, 0, 46, 530, 15, 0, 0, 0, 1, 7, 0, 0, 3, 0, 6, 71, 2, 0, 0, 0, 125, 512, 0, 0, 1269, 0, 5479, 149732, 500, 0, 0, 0, 6, 34, 0, 0, 21, 0, 47, 498, 11 +2,2, 1, 0, 0, 0, 3, 18, 15, 0, 1, 0, 22, 89, 12, 0, 0, 0, 3, 5562, 10635, 0, 2, 0, 3165, 22810, 136140, 0, 0, 0, 3, 54, 96, 0, 1, 0, 80, 404, 129, 0, 0, 0, 3, 18, 15, 0, 1, 0, 22, 89, 12, 0, 0, 0, 36, 10323, 2676, 0, 579, 0, 786, 61206, 226845, 0, 0, 0, 9, 60, 84, 0, 9, 0, 92, 588, 129 +26,2, 1, 0, 0, 0, 51, 252, 45, 0, 8, 0, 27, 731, 108, 0, 0, 0, 24228, 110277, 11448, 0, 35, 0, 11838, 420698, 312936, 0, 0, 0, 324, 1527, 279, 0, 12, 0, 198, 5041, 861, 0, 0, 0, 51, 252, 45, 0, 8, 0, 27, 731, 108, 0, 0, 0, 50289, 168120, 19464, 0, 3103, 0, 8874, 597306, 604125, 0, 0, 0, 339, 1575, 291, 0, 59, 0, 186, 5299, 882 +27,2, 2, 0, 0, 0, 200, 204, 176, 0, 4, 0, 84, 557, 176, 0, 0, 0, 100150, 117630, 80620, 0, 65, 0, 58636, 761772, 130030, 0, 0, 0, 1432, 1456, 1188, 0, 6, 0, 590, 3911, 1260, 0, 0, 0, 200, 204, 176, 0, 4, 0, 84, 557, 176, 0, 0, 0, 2499426, 339686, 605394, 0, 3988, 0, 88704, 1534262, 262312, 0, 0, 0, 1548, 1488, 1274, 0, 23, 0, 626, 4231, 1400 +2,2, 2, 0, 0, 0, 36, 12, 15, 0, 0, 0, 2, 22, 26, 0, 0, 0, 16882, 10152, 6049, 0, 0, 0, 2, 7204, 129712, 0, 0, 0, 262, 102, 104, 0, 0, 0, 2, 126, 180, 0, 0, 0, 36, 12, 15, 0, 0, 0, 2, 22, 26, 0, 0, 0, 34974, 7392, 7004, 0, 0, 0, 8, 16368, 149362, 0, 0, 0, 286, 98, 98, 0, 0, 0, 4, 170, 224 +26,2, 3, 0, 0, 0, 2, 10, 0, 0, 3, 0, 13, 28, 1, 0, 0, 0, 1151, 5156, 0, 0, 195, 0, 14793, 20547, 1994, 0, 0, 0, 14, 73, 0, 0, 18, 0, 112, 112, 10, 0, 0, 0, 2, 10, 0, 0, 3, 0, 13, 28, 1, 0, 0, 0, 1985, 8747, 0, 0, 11508, 0, 5772, 24774, 1071, 0, 0, 0, 13, 79, 0, 0, 33, 0, 91, 136, 10 +27,2, 1, 0, 0, 0, 93, 516, 144, 0, 4, 0, 354, 1917, 303, 0, 0, 0, 17115, 346134, 55971, 0, 158, 0, 317277, 2103792, 561921, 0, 0, 0, 303, 3261, 861, 0, 9, 0, 2562, 13804, 2310, 0, 0, 0, 93, 516, 144, 0, 4, 0, 354, 1917, 303, 0, 0, 0, 18444, 576120, 146238, 0, 1648, 0, 506919, 5038145, 4586901, 0, 0, 0, 351, 3339, 945, 0, 31, 0, 2688, 14784, 2469 +2,2, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 1068, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 2038, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 31, 0 +28,2, 3, 0, 0, 0, 5, 3, 0, 1, 4, 0, 1, 27, 5, 0, 0, 0, 2723, 135, 0, 25, 68, 0, 6, 13354, 21247, 0, 0, 0, 40, 13, 0, 4, 8, 0, 2, 171, 52, 0, 0, 0, 5, 3, 0, 1, 4, 0, 1, 27, 5, 0, 0, 0, 1460, 12974, 0, 140, 4898, 0, 142, 14221, 167428, 0, 0, 0, 36, 25, 0, 7, 24, 0, 7, 167, 71 +28,2, 2, 0, 0, 0, 113, 42, 74, 0, 1, 0, 116, 281, 148, 0, 0, 0, 55554, 25378, 40384, 0, 106, 0, 52654, 467138, 121514, 0, 0, 0, 722, 296, 474, 0, 6, 0, 748, 2144, 1140, 0, 0, 0, 113, 42, 74, 0, 1, 0, 116, 281, 148, 0, 0, 0, 71919, 172268, 58134, 0, 163, 0, 48276, 786713, 242018, 0, 0, 0, 697, 310, 476, 0, 7, 0, 650, 2233, 1210 +2,2, 1, 0, 0, 0, 3, 22, 9, 0, 0, 0, 24, 114, 15, 0, 0, 0, 0, 8518, 1719, 0, 0, 0, 11235, 44559, 2133, 0, 0, 0, 0, 104, 36, 0, 0, 0, 93, 531, 69, 0, 0, 0, 3, 22, 9, 0, 0, 0, 24, 114, 15, 0, 0, 0, 12, 7357, 2244, 0, 0, 0, 4134, 83781, 3819, 0, 0, 0, 6, 102, 60, 0, 0, 0, 141, 813, 93 +27,2, 3, 1, 1, 0, 13, 43, 2, 3, 18, 0, 55, 462, 8, 45, 98, 0, 5599, 56140, 501, 912, 627, 0, 52371, 522506, 12029, 5, 6, 0, 79, 264, 10, 17, 49, 0, 330, 3053, 70, 1, 1, 0, 13, 43, 2, 3, 18, 0, 55, 462, 8, 1768, 44, 0, 4409, 77141, 349, 4073, 42366, 0, 101616, 833279, 13605, 10, 5, 0, 89, 321, 14, 25, 140, 0, 403, 3231, 60 +2,2, 2, 0, 0, 0, 32, 17, 14, 0, 0, 0, 10, 38, 12, 0, 0, 0, 8754, 20830, 5902, 0, 0, 0, 11010, 45376, 38724, 0, 0, 0, 170, 153, 98, 0, 0, 0, 82, 304, 118, 0, 0, 0, 32, 17, 14, 0, 0, 0, 10, 38, 12, 0, 0, 0, 13534, 36052, 3966, 0, 0, 0, 21602, 48084, 43772, 0, 0, 0, 184, 162, 88, 0, 0, 0, 98, 308, 126 +27,2, 2, 0, 0, 0, 819, 506, 728, 0, 20, 0, 466, 1692, 1064, 0, 0, 0, 995267, 418112, 444900, 0, 987, 0, 463330, 1928859, 825780, 0, 0, 0, 6076, 3762, 4910, 0, 21, 0, 3264, 11689, 7616, 0, 0, 0, 819, 506, 728, 0, 20, 0, 466, 1692, 1064, 0, 0, 0, 8376448, 6721944, 1741062, 0, 57391, 0, 4006104, 33284779, 2555632, 0, 0, 0, 6335, 3706, 5174, 0, 130, 0, 3174, 12652, 8010 +28,2, 1, 0, 0, 0, 57, 246, 66, 0, 0, 0, 72, 1054, 219, 0, 0, 0, 22530, 156315, 8298, 0, 0, 0, 201531, 1271696, 205299, 0, 0, 0, 270, 1407, 282, 0, 0, 0, 453, 7499, 1692, 0, 0, 0, 57, 246, 66, 0, 0, 0, 72, 1054, 219, 0, 0, 0, 23613, 534948, 28302, 0, 0, 0, 3542859, 1862805, 1421673, 0, 0, 0, 270, 1518, 210, 0, 0, 0, 531, 8060, 1857 +2,2, 3, 0, 0, 0, 0, 4, 1, 0, 0, 0, 1, 80, 0, 0, 0, 0, 0, 615, 52, 0, 0, 0, 125, 39790, 0, 0, 0, 0, 0, 18, 5, 0, 0, 0, 6, 537, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 1, 80, 0, 0, 0, 0, 0, 900, 5, 0, 0, 0, 176, 445525, 0, 0, 0, 0, 0, 27, 2, 0, 0, 0, 7, 666, 0 +27,2, 1, 0, 0, 0, 426, 1458, 303, 0, 23, 0, 846, 6143, 789, 0, 0, 0, 223500, 969735, 216576, 0, 4754, 0, 722748, 6286067, 1249164, 0, 0, 0, 2292, 8940, 1908, 0, 88, 0, 6183, 44688, 5799, 0, 0, 0, 426, 1458, 303, 0, 23, 0, 846, 6143, 789, 0, 0, 0, 533280, 2099079, 3133356, 0, 16118, 0, 8972823, 11835932, 2917317, 0, 0, 0, 2502, 9450, 2031, 0, 185, 0, 6504, 47564, 6645 +1,3, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5043, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43872, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0 +3,2, 1, 0, 1, 0, 66, 152, 69, 0, 3, 0, 138, 1336, 423, 0, 148, 0, 37956, 66290, 20412, 0, 151, 0, 71919, 1485564, 400605, 0, 7, 0, 333, 827, 414, 0, 16, 0, 774, 9452, 3159, 0, 1, 0, 66, 152, 69, 0, 3, 0, 138, 1336, 423, 0, 71, 0, 50217, 72292, 17997, 0, 7611, 0, 110412, 2488026, 630654, 0, 6, 0, 354, 750, 378, 0, 29, 0, 906, 10455, 3279 +28,2, 3, 0, 0, 0, 9, 28, 5, 2, 40, 0, 22, 536, 33, 0, 0, 0, 9132, 3737, 4159, 50, 4085, 0, 13536, 534613, 23317, 0, 0, 0, 75, 164, 36, 8, 120, 0, 156, 3614, 205, 0, 0, 0, 9, 28, 5, 2, 40, 0, 22, 536, 33, 0, 0, 0, 9673, 5748, 4801, 278, 49958, 0, 13911, 639312, 23282, 0, 0, 0, 76, 177, 43, 14, 302, 0, 147, 3475, 217 +28,2, 2, 0, 0, 0, 650, 776, 572, 0, 30, 0, 703, 1542, 866, 0, 0, 0, 383132, 657612, 441816, 0, 1894, 0, 485779, 2660958, 839408, 0, 0, 0, 4684, 6070, 3776, 0, 63, 0, 4600, 11079, 6424, 0, 0, 0, 650, 776, 572, 0, 30, 0, 703, 1542, 866, 0, 0, 0, 947042, 3236722, 2801774, 0, 63960, 0, 1089620, 9422862, 2017050, 0, 0, 0, 4792, 6138, 4032, 0, 185, 0, 3832, 12001, 6752 +2,2, 1, 0, 0, 0, 33, 45, 18, 0, 0, 0, 33, 268, 51, 0, 0, 0, 3831, 7974, 6924, 0, 0, 0, 759, 205783, 31023, 0, 0, 0, 126, 147, 87, 0, 0, 0, 60, 1936, 375, 0, 0, 0, 33, 45, 18, 0, 0, 0, 33, 268, 51, 0, 0, 0, 8889, 1566219, 4218, 0, 0, 0, 2373, 276716, 58764, 0, 0, 0, 150, 186, 96, 0, 0, 0, 159, 2109, 432 +3,2, 2, 0, 0, 0, 135, 128, 80, 0, 1, 0, 62, 389, 74, 0, 0, 0, 56939, 79410, 28208, 0, 65, 0, 30150, 425711, 66356, 0, 0, 0, 968, 954, 558, 0, 6, 0, 416, 2642, 448, 0, 0, 0, 135, 128, 80, 0, 1, 0, 62, 389, 74, 0, 0, 0, 99762, 155064, 38632, 0, 3686, 0, 804002, 778716, 99956, 0, 0, 0, 920, 952, 574, 0, 11, 0, 386, 2745, 438 +2,2, 2, 0, 0, 0, 35, 22, 22, 0, 0, 0, 10, 124, 42, 0, 0, 0, 17006, 12096, 9064, 0, 0, 0, 5502, 313484, 373676, 0, 0, 0, 257, 148, 158, 0, 0, 0, 64, 858, 330, 0, 0, 0, 35, 22, 22, 0, 0, 0, 10, 124, 42, 0, 0, 0, 252979, 41222, 9124, 0, 0, 0, 2566, 487398, 403232, 0, 0, 0, 266, 182, 166, 0, 0, 0, 56, 1100, 344 +1,3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0 +3,2, 3, 0, 0, 0, 10, 8, 0, 0, 6, 0, 3, 41, 6, 0, 0, 0, 6401, 1734, 0, 0, 150, 0, 337, 13490, 7485, 0, 0, 0, 84, 50, 0, 0, 19, 0, 14, 198, 41, 0, 0, 0, 10, 8, 0, 0, 6, 0, 3, 41, 6, 0, 0, 0, 27461, 2087, 0, 0, 76550, 0, 2241, 82778, 14082, 0, 0, 0, 87, 51, 0, 0, 62, 0, 16, 221, 56 +2,2, 3, 0, 0, 0, 1, 3, 0, 0, 0, 0, 1, 31, 2, 0, 0, 0, 68, 2634, 0, 0, 0, 0, 782, 14970, 1353, 0, 0, 0, 6, 24, 0, 0, 0, 0, 9, 205, 17, 0, 0, 0, 1, 3, 0, 0, 0, 0, 1, 31, 2, 0, 0, 0, 8, 3897, 0, 0, 0, 0, 515, 721397, 1406, 0, 0, 0, 3, 25, 0, 0, 0, 0, 9, 247, 18 +28,2, 1, 0, 2, 0, 366, 1027, 399, 0, 25, 0, 372, 5895, 738, 0, 924, 0, 206493, 858333, 290004, 0, 2150, 0, 453396, 6317170, 963867, 0, 16, 0, 1785, 6173, 2619, 0, 50, 0, 2592, 42035, 5586, 0, 2, 0, 366, 1027, 399, 0, 25, 0, 372, 5895, 738, 0, 4710, 0, 326721, 1494597, 1949316, 0, 69096, 0, 778062, 15836547, 1551159, 0, 20, 0, 1902, 6535, 2913, 0, 209, 0, 2874, 45532, 5880 +17,1, 2, 1, 3, 0, 708, 197, 184, 0, 55, 0, 374, 1169, 808, 77, 2483, 0, 913718, 255077, 116070, 0, 9434, 0, 500410, 1566332, 695306, 6, 24, 0, 5802, 1484, 1280, 0, 191, 0, 2596, 7945, 5624, 1, 3, 0, 708, 197, 184, 0, 55, 0, 374, 1169, 808, 1220, 12651, 0, 1265811, 1554037, 207960, 0, 584582, 0, 1789834, 3342624, 8206496, 10, 28, 0, 5547, 1530, 1294, 0, 414, 0, 2996, 9334, 6668 +16,1, 2, 4, 18, 0, 822, 535, 498, 6, 156, 0, 1264, 3446, 1270, 5407, 8762, 0, 807183, 534159, 421202, 2027, 35936, 0, 2528693, 4095084, 1989942, 33, 129, 0, 5937, 4108, 3200, 31, 541, 0, 9779, 23839, 9374, 4, 18, 0, 822, 535, 498, 6, 156, 0, 1264, 3446, 1270, 20470, 35799, 0, 2275316, 2434790, 1660594, 10169, 1647892, 0, 10073865, 17383546, 7051606, 46, 156, 0, 6072, 4416, 3382, 51, 1199, 0, 10593, 27593, 10472 +16,1, 1, 0, 1, 0, 321, 839, 189, 0, 10, 0, 117, 2835, 654, 0, 219, 0, 87159, 255825, 130488, 0, 6319, 0, 69108, 3360665, 1096482, 0, 7, 0, 1143, 3683, 1242, 0, 36, 0, 801, 21425, 5004, 0, 1, 0, 321, 839, 189, 0, 10, 0, 117, 2835, 654, 0, 225, 0, 137784, 405198, 563604, 0, 65264, 0, 123426, 5101314, 1559922, 0, 7, 0, 1254, 3830, 1380, 0, 86, 0, 900, 22166, 5313 +17,1, 3, 3, 8, 0, 17, 129, 3, 0, 56, 0, 81, 305, 67, 2275, 6172, 0, 6374, 19656, 169, 0, 19402, 0, 231462, 219534, 66378, 25, 67, 0, 92, 291, 12, 0, 236, 0, 553, 1763, 420, 3, 8, 0, 17, 129, 3, 0, 56, 0, 81, 305, 67, 3217, 16348, 0, 6054, 30947, 653, 0, 386210, 0, 342413, 595550, 616799, 28, 72, 0, 121, 354, 21, 0, 492, 0, 676, 2217, 505 +16,1, 3, 1, 0, 0, 2, 29, 1, 1, 21, 0, 19, 101, 13, 14, 0, 0, 772, 11433, 94, 1, 317, 0, 27801, 40000, 18212, 3, 0, 0, 16, 182, 6, 1, 29, 0, 118, 500, 105, 1, 0, 0, 2, 29, 1, 1, 21, 0, 19, 101, 13, 10, 0, 0, 755, 13209, 44, 945, 104257, 0, 31739, 115514, 87852, 3, 0, 0, 16, 187, 5, 9, 158, 0, 147, 670, 113 +17,1, 1, 0, 9, 0, 135, 414, 82, 0, 20, 0, 495, 3118, 318, 0, 2094, 0, 40968, 136143, 47196, 0, 8039, 0, 461541, 2695075, 497811, 0, 57, 0, 594, 2037, 492, 0, 57, 0, 3753, 21504, 2220, 0, 9, 0, 135, 414, 82, 0, 20, 0, 495, 3118, 318, 0, 48064, 0, 107997, 270197, 52716, 0, 44117, 0, 904653, 5367427, 1226064, 0, 81, 0, 645, 2097, 538, 0, 148, 0, 3948, 23810, 2439 +4,2, 1, 0, 0, 0, 24, 216, 27, 0, 7, 0, 42, 378, 48, 0, 0, 0, 4950, 77313, 22950, 0, 1995, 0, 184377, 254803, 69744, 0, 0, 0, 105, 1392, 159, 0, 18, 0, 348, 2423, 396, 0, 0, 0, 24, 216, 27, 0, 7, 0, 42, 378, 48, 0, 0, 0, 5880, 140727, 56841, 0, 40408, 0, 213963, 535776, 71595, 0, 0, 0, 117, 1443, 177, 0, 69, 0, 414, 2925, 396 +3,2, 1, 0, 0, 0, 342, 1278, 471, 0, 22, 0, 414, 5157, 900, 0, 0, 0, 208674, 594759, 1348932, 0, 1597, 0, 328488, 6688960, 1902720, 0, 0, 0, 1839, 7917, 3465, 0, 55, 0, 2346, 35502, 6963, 0, 0, 0, 342, 1278, 471, 0, 22, 0, 414, 5157, 900, 0, 0, 0, 1216527, 6798900, 7543281, 0, 80951, 0, 924390, 11471051, 2815812, 0, 0, 0, 1992, 7662, 3291, 0, 170, 0, 2946, 38972, 7287 +4,2, 2, 0, 0, 0, 79, 26, 30, 0, 3, 0, 64, 295, 66, 0, 0, 0, 44560, 6074, 25996, 0, 6, 0, 94306, 435544, 61064, 0, 0, 0, 561, 158, 230, 0, 2, 0, 436, 2064, 442, 0, 0, 0, 79, 26, 30, 0, 3, 0, 64, 295, 66, 0, 0, 0, 58469, 14966, 387814, 0, 585, 0, 133946, 1048291, 367418, 0, 0, 0, 558, 158, 246, 0, 22, 0, 426, 2206, 462 +3,2, 2, 0, 0, 0, 717, 2836, 476, 0, 21, 0, 358, 1733, 660, 0, 0, 0, 511328, 480930, 270176, 0, 3975, 0, 351302, 2055603, 741344, 0, 0, 0, 5203, 10972, 3292, 0, 61, 0, 2450, 11411, 4626, 0, 0, 0, 717, 2836, 476, 0, 21, 0, 358, 1733, 660, 0, 0, 0, 9901743, 1927156, 1060676, 0, 23335, 0, 2212734, 3514153, 2640210, 0, 0, 0, 5178, 11298, 3312, 0, 154, 0, 2492, 12226, 4840 +4,2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4650, 20401, 10191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 123, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 27, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 15481, 67548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 119, 16 +3,2, 3, 0, 0, 0, 10, 106, 7, 0, 35, 0, 280, 387, 26, 0, 0, 0, 16464, 16756, 11466, 0, 5605, 0, 300984, 332202, 29020, 0, 0, 0, 81, 363, 63, 0, 90, 0, 2060, 1707, 149, 0, 0, 0, 10, 106, 7, 0, 35, 0, 280, 387, 26, 0, 0, 0, 16159, 21787, 13722, 0, 69011, 0, 264297, 1243640, 117377, 0, 0, 0, 82, 325, 65, 0, 270, 0, 1842, 2063, 178 +5,2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1761, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10 +4,2, 1, 0, 0, 0, 471, 1668, 438, 0, 28, 0, 315, 5961, 1179, 0, 0, 0, 256215, 1125459, 476676, 0, 7462, 0, 296043, 7268746, 2085357, 0, 0, 0, 2757, 10464, 3096, 0, 135, 0, 2325, 44367, 8862, 0, 0, 0, 471, 1668, 438, 0, 28, 0, 315, 5961, 1179, 0, 0, 0, 2297709, 2935329, 991440, 0, 26958, 0, 405324, 24268783, 5170863, 0, 0, 0, 3003, 10812, 3024, 0, 225, 0, 2391, 46246, 9342 +5,2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11503, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0 +4,2, 2, 0, 0, 0, 1056, 684, 782, 0, 33, 0, 804, 2368, 946, 0, 0, 0, 809248, 768694, 553750, 0, 20174, 0, 804472, 2059470, 1566550, 0, 0, 0, 7766, 5628, 5868, 0, 121, 0, 5606, 15627, 6628, 0, 0, 0, 1056, 684, 782, 0, 33, 0, 804, 2368, 946, 0, 0, 0, 7149848, 5099222, 1840820, 0, 33473, 0, 7026972, 12366648, 14762270, 0, 0, 0, 7960, 5846, 5974, 0, 242, 0, 5678, 17786, 7328 +4,2, 3, 0, 0, 0, 6, 35, 5, 3, 23, 0, 115, 234, 16, 0, 0, 0, 31498, 53114, 9296, 1, 1647, 0, 153332, 99980, 117463, 0, 0, 0, 57, 207, 44, 1, 77, 0, 860, 791, 86, 0, 0, 0, 6, 35, 5, 3, 23, 0, 115, 234, 16, 0, 0, 0, 45547, 59104, 12570, 1279, 71775, 0, 189920, 180450, 123385, 0, 0, 0, 57, 236, 42, 23, 192, 0, 789, 988, 97 +5,2, 2, 1, 0, 0, 873, 560, 966, 0, 29, 0, 632, 2655, 651, 8, 0, 0, 702934, 574096, 776990, 0, 5536, 0, 1029212, 3550172, 764118, 3, 0, 0, 6535, 4280, 7212, 0, 72, 0, 4470, 18682, 4567, 1, 0, 0, 873, 560, 966, 0, 29, 0, 632, 2655, 651, 0, 0, 0, 6631558, 6317234, 14488952, 0, 112562, 0, 2653512, 7002146, 2151462, 0, 0, 0, 6672, 4502, 7390, 0, 215, 0, 4442, 20009, 4677 +4,2, 1, 0, 0, 0, 15, 42, 12, 0, 1, 0, 9, 228, 51, 0, 0, 0, 771, 19509, 3300, 0, 21, 0, 3609, 259320, 36132, 0, 0, 0, 42, 318, 75, 0, 4, 0, 39, 1792, 390, 0, 0, 0, 15, 42, 12, 0, 1, 0, 9, 228, 51, 0, 0, 0, 1770, 31209, 4533, 0, 241, 0, 10566, 334217, 27039, 0, 0, 0, 63, 321, 78, 0, 7, 0, 51, 1873, 351 +5,2, 1, 0, 0, 0, 480, 1209, 345, 0, 13, 0, 690, 5610, 957, 0, 0, 0, 229368, 1205172, 186957, 0, 7883, 0, 613413, 7694302, 1536192, 0, 0, 0, 2775, 7191, 2160, 0, 64, 0, 4314, 40526, 7596, 0, 0, 0, 480, 1209, 345, 0, 13, 0, 690, 5610, 957, 0, 0, 0, 500199, 6125073, 279174, 0, 46524, 0, 2858907, 18378369, 2699526, 0, 0, 0, 2985, 7476, 2154, 0, 108, 0, 4671, 42683, 7893 +4,2, 2, 0, 0, 0, 16, 6, 4, 0, 1, 0, 40, 64, 36, 0, 0, 0, 9048, 2904, 1120, 0, 0, 0, 16356, 52674, 14888, 0, 0, 0, 102, 48, 26, 0, 0, 0, 228, 424, 228, 0, 0, 0, 16, 6, 4, 0, 1, 0, 40, 64, 36, 0, 0, 0, 10796, 1174, 5214, 0, 13, 0, 17000, 58712, 61620, 0, 0, 0, 114, 38, 24, 0, 3, 0, 232, 443, 214 +6,2, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0 +4,2, 3, 0, 0, 0, 1, 3, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 427, 2639, 0, 0, 195, 0, 0, 220, 0, 0, 0, 0, 8, 27, 0, 0, 18, 0, 0, 7, 0, 0, 0, 0, 1, 3, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 525, 13459, 0, 0, 11074, 0, 0, 175, 0, 0, 0, 0, 9, 31, 0, 0, 36, 0, 0, 7, 0 +5,2, 3, 0, 0, 0, 12, 24, 4, 2, 27, 0, 74, 352, 22, 0, 0, 0, 3626, 17800, 1333, 2, 2405, 0, 53852, 368038, 116724, 0, 0, 0, 81, 176, 29, 1, 74, 0, 472, 1779, 74, 0, 0, 0, 12, 24, 4, 2, 27, 0, 74, 352, 22, 0, 0, 0, 5377, 19822, 1202, 315, 103670, 0, 53533, 573595, 122814, 0, 0, 0, 82, 178, 25, 11, 210, 0, 420, 1906, 131 +5,2, 2, 0, 0, 0, 114, 74, 68, 0, 2, 0, 90, 267, 76, 0, 0, 0, 44158, 27702, 45050, 0, 71, 0, 432028, 323036, 180252, 0, 0, 0, 826, 520, 448, 0, 8, 0, 648, 1731, 554, 0, 0, 0, 114, 74, 68, 0, 2, 0, 90, 267, 76, 0, 0, 0, 121212, 96998, 75160, 0, 3911, 0, 1743964, 572356, 627626, 0, 0, 0, 810, 536, 482, 0, 18, 0, 736, 1882, 578 +5,2, 1, 0, 0, 0, 81, 181, 84, 0, 5, 0, 93, 976, 174, 0, 0, 0, 67311, 126098, 30177, 0, 239, 0, 62118, 803089, 195582, 0, 0, 0, 552, 1042, 552, 0, 23, 0, 543, 6799, 1389, 0, 0, 0, 81, 181, 84, 0, 5, 0, 93, 976, 174, 0, 0, 0, 152409, 177586, 29037, 0, 24224, 0, 70293, 1635574, 332115, 0, 0, 0, 618, 1045, 435, 0, 49, 0, 636, 7406, 1395 +6,2, 3, 0, 0, 0, 7, 23, 5, 1, 24, 0, 60, 348, 13, 0, 0, 0, 3873, 32168, 6499, 0, 2594, 0, 91153, 186033, 262, 0, 0, 0, 53, 159, 47, 0, 79, 0, 472, 1646, 23, 0, 0, 0, 7, 23, 5, 1, 24, 0, 60, 348, 13, 0, 0, 0, 5686, 41661, 6038, 14, 157850, 0, 86350, 1423118, 2250, 0, 0, 0, 55, 180, 45, 3, 184, 0, 449, 1728, 55 +6,2, 1, 0, 2, 0, 318, 1012, 372, 0, 25, 0, 1542, 4638, 651, 0, 76, 0, 161388, 677879, 251916, 0, 8521, 0, 1100193, 7509842, 1118289, 0, 10, 0, 1971, 6230, 2547, 0, 127, 0, 11127, 35519, 5151, 0, 2, 0, 318, 1012, 372, 0, 25, 0, 1542, 4638, 651, 0, 24, 0, 255738, 1517097, 1583862, 0, 1772111, 0, 5287935, 17894631, 1709439, 0, 6, 0, 2106, 6495, 2547, 0, 211, 0, 11568, 36760, 5439 +6,2, 2, 0, 0, 0, 865, 382, 438, 0, 32, 0, 554, 1640, 806, 0, 0, 0, 604397, 382916, 325194, 0, 3907, 0, 467014, 1271261, 984242, 0, 0, 0, 6475, 2914, 3340, 0, 52, 0, 3550, 9970, 5444, 0, 0, 0, 865, 382, 438, 0, 32, 0, 554, 1640, 806, 0, 0, 0, 11239619, 7127702, 3599602, 0, 69166, 0, 1000866, 26945968, 9531876, 0, 0, 0, 6601, 3100, 3560, 0, 223, 0, 3428, 11175, 5744 +5,2, 3, 0, 0, 0, 2, 4, 0, 0, 5, 0, 2, 42, 3, 0, 0, 0, 993, 4019, 0, 0, 73, 0, 186, 101614, 705, 0, 0, 0, 17, 34, 0, 0, 10, 0, 12, 116, 15, 0, 0, 0, 2, 4, 0, 0, 5, 0, 2, 42, 3, 0, 0, 0, 850, 3410, 0, 0, 5155, 0, 109, 135437, 739, 0, 0, 0, 17, 34, 0, 0, 38, 0, 10, 147, 11 +7,2, 1, 0, 1, 0, 285, 797, 369, 0, 31, 0, 351, 4394, 958, 0, 271, 0, 121017, 590018, 186978, 0, 5122, 0, 303345, 5829791, 2097099, 0, 8, 0, 1581, 4945, 2466, 0, 91, 0, 2235, 31517, 7021, 0, 1, 0, 285, 797, 369, 0, 31, 0, 351, 4394, 958, 0, 268, 0, 171348, 2295563, 325287, 0, 68875, 0, 886896, 12839963, 3958433, 0, 8, 0, 1650, 5134, 2235, 0, 221, 0, 2538, 34552, 7632 +7,2, 2, 0, 0, 0, 528, 172, 240, 0, 6, 0, 204, 913, 294, 0, 0, 0, 322154, 111622, 308936, 0, 1459, 0, 255748, 1048415, 546948, 0, 0, 0, 3956, 1290, 1894, 0, 27, 0, 1348, 5758, 1874, 0, 0, 0, 528, 172, 240, 0, 6, 0, 204, 913, 294, 0, 0, 0, 784748, 919638, 2593298, 0, 50301, 0, 1645180, 2955775, 1854450, 0, 0, 0, 3886, 1268, 1944, 0, 54, 0, 1456, 6331, 1926 +7,2, 3, 0, 0, 0, 6, 20, 2, 3, 14, 0, 21, 152, 5, 0, 0, 0, 2046, 7877, 2943, 2638, 3040, 0, 17120, 64404, 141994, 0, 0, 0, 44, 129, 16, 19, 54, 0, 136, 569, 24, 0, 0, 0, 6, 20, 2, 3, 14, 0, 21, 152, 5, 0, 0, 0, 3008, 71031, 2779, 14189, 18514, 0, 26877, 207888, 230587, 0, 0, 0, 46, 140, 16, 31, 108, 0, 147, 644, 45 +6,2, 3, 0, 0, 0, 1, 17, 0, 0, 15, 0, 23, 113, 40, 0, 0, 0, 85, 1764, 0, 0, 205, 0, 32307, 92410, 6542, 0, 0, 0, 6, 90, 0, 0, 24, 0, 196, 491, 139, 0, 0, 0, 1, 17, 0, 0, 15, 0, 23, 113, 40, 0, 0, 0, 176, 3283, 0, 0, 34153, 0, 27321, 148202, 9098, 0, 0, 0, 7, 108, 0, 0, 116, 0, 165, 546, 205 +6,2, 1, 0, 0, 0, 204, 360, 93, 0, 31, 0, 345, 2550, 297, 0, 0, 0, 107301, 276285, 32175, 0, 12183, 0, 630330, 2878226, 336225, 0, 0, 0, 1344, 1818, 567, 0, 177, 0, 2454, 18096, 2127, 0, 0, 0, 204, 360, 93, 0, 31, 0, 345, 2550, 297, 0, 0, 0, 108858, 440361, 28245, 0, 26477, 0, 758862, 5473630, 714918, 0, 0, 0, 1299, 1962, 486, 0, 231, 0, 2628, 19383, 2394 +6,2, 2, 0, 0, 0, 284, 114, 162, 0, 9, 0, 290, 584, 298, 0, 0, 0, 158178, 85998, 117284, 0, 92, 0, 176350, 684903, 315414, 0, 0, 0, 2080, 834, 1104, 0, 13, 0, 1492, 3565, 1710, 0, 0, 0, 284, 114, 162, 0, 9, 0, 290, 584, 298, 0, 0, 0, 2080824, 150622, 2087682, 0, 4625, 0, 2788010, 1218706, 1540318, 0, 0, 0, 2016, 810, 1174, 0, 63, 0, 1280, 3845, 1560 +7,2, 1, 0, 0, 0, 255, 708, 820, 0, 30, 0, 504, 4371, 894, 0, 0, 0, 166560, 595968, 276469, 0, 4676, 0, 635142, 4897075, 2655972, 0, 0, 0, 1707, 4368, 2878, 0, 101, 0, 3357, 30385, 6804, 0, 0, 0, 255, 708, 820, 0, 30, 0, 504, 4371, 894, 0, 0, 0, 583839, 1166196, 520332, 0, 57921, 0, 1128717, 25193013, 3677055, 0, 0, 0, 1794, 4365, 2072, 0, 220, 0, 3930, 34556, 6987 +7,2, 2, 0, 0, 0, 447, 232, 278, 2, 19, 0, 430, 1133, 408, 0, 0, 0, 244374, 142950, 146492, 0, 379, 0, 150264, 831999, 293184, 0, 0, 0, 3064, 1746, 1942, 0, 25, 0, 2026, 6947, 2210, 0, 0, 0, 447, 232, 278, 2, 19, 0, 430, 1133, 408, 0, 0, 0, 397610, 2396798, 321610, 167, 5635, 0, 300383, 25038591, 563844, 0, 0, 0, 2999, 1828, 1996, 7, 112, 0, 1549, 7248, 2056 +8,2, 3, 0, 0, 0, 0, 9, 6, 0, 1, 0, 1, 99, 1, 0, 0, 0, 0, 22940, 4885, 0, 65, 0, 28, 61264, 1791, 0, 0, 0, 0, 68, 52, 0, 6, 0, 4, 382, 10, 0, 0, 0, 0, 9, 6, 0, 1, 0, 1, 99, 1, 0, 0, 0, 0, 29439, 2714, 0, 3734, 0, 19, 142597, 1480, 0, 0, 0, 0, 71, 46, 0, 11, 0, 4, 418, 10 +7,2, 3, 0, 0, 0, 3, 27, 5, 0, 10, 0, 7, 184, 13, 0, 0, 0, 916, 11906, 1602, 0, 2748, 0, 3068, 240719, 118075, 0, 0, 0, 14, 148, 37, 0, 39, 0, 47, 786, 51, 0, 0, 0, 3, 27, 5, 0, 10, 0, 7, 184, 13, 0, 0, 0, 354, 11353, 4802, 0, 10214, 0, 2844, 375802, 115504, 0, 0, 0, 18, 143, 23, 0, 74, 0, 43, 844, 56 +8,2, 2, 0, 0, 0, 160, 116, 224, 0, 0, 0, 108, 339, 166, 0, 0, 0, 66866, 86704, 126696, 0, 0, 0, 53836, 945567, 236722, 0, 0, 0, 1048, 912, 1710, 0, 0, 0, 634, 2163, 980, 0, 0, 0, 160, 116, 224, 0, 0, 0, 108, 339, 166, 0, 0, 0, 125122, 103434, 171666, 0, 0, 0, 50212, 1096448, 858696, 0, 0, 0, 1040, 906, 1600, 0, 0, 0, 506, 2112, 924 +8,2, 1, 0, 0, 0, 111, 219, 123, 0, 0, 0, 81, 694, 228, 0, 0, 0, 40635, 78165, 43290, 0, 0, 0, 74394, 597792, 320070, 0, 0, 0, 390, 936, 654, 0, 0, 0, 486, 4826, 1575, 0, 0, 0, 111, 219, 123, 0, 0, 0, 81, 694, 228, 0, 0, 0, 124113, 117744, 95061, 0, 0, 0, 96987, 958837, 1131129, 0, 0, 0, 486, 978, 675, 0, 0, 0, 528, 5285, 1671 +9,2, 2, 0, 0, 0, 260, 100, 232, 0, 7, 0, 26, 418, 212, 0, 0, 0, 151162, 123606, 110182, 0, 229, 0, 18896, 326839, 154020, 0, 0, 0, 1836, 744, 1650, 0, 10, 0, 192, 2806, 1360, 0, 0, 0, 260, 100, 232, 0, 7, 0, 26, 418, 212, 0, 0, 0, 353640, 127910, 150158, 0, 1797, 0, 23316, 560404, 345342, 0, 0, 0, 1810, 712, 1534, 0, 44, 0, 194, 2916, 1320 +9,2, 1, 0, 0, 0, 99, 432, 105, 0, 4, 0, 108, 834, 159, 0, 0, 0, 58329, 295836, 100305, 0, 2005, 0, 94512, 885682, 393486, 0, 0, 0, 528, 2652, 609, 0, 18, 0, 810, 5579, 1290, 0, 0, 0, 99, 432, 105, 0, 4, 0, 108, 834, 159, 0, 0, 0, 151047, 587370, 171996, 0, 3275, 0, 607989, 3084412, 1028220, 0, 0, 0, 585, 2721, 648, 0, 33, 0, 882, 5921, 1329 +8,2, 3, 0, 0, 0, 2, 16, 8, 0, 12, 0, 9, 196, 4, 0, 0, 0, 2711, 5067, 7053, 0, 4, 0, 3930, 78454, 2303, 0, 0, 0, 18, 89, 61, 0, 4, 0, 26, 770, 18, 0, 0, 0, 2, 16, 8, 0, 12, 0, 9, 196, 4, 0, 0, 0, 7434, 4238, 6833, 0, 4774, 0, 3913, 134798, 1799, 0, 0, 0, 17, 101, 56, 0, 80, 0, 26, 912, 16 +8,2, 2, 1, 1, 0, 469, 201, 734, 0, 3, 0, 368, 1008, 526, 4, 4, 0, 206646, 125218, 428664, 0, 0, 0, 668192, 1547236, 899102, 2, 2, 0, 3142, 1462, 5694, 0, 0, 0, 1820, 5656, 3174, 1, 1, 0, 469, 201, 734, 0, 3, 0, 368, 1008, 526, 9, 9, 0, 2288633, 257643, 2622498, 0, 177, 0, 654996, 1829802, 1268408, 3, 3, 0, 3129, 1487, 5376, 0, 13, 0, 1200, 5505, 3100 +8,2, 1, 0, 0, 0, 222, 537, 184, 0, 17, 0, 234, 1693, 351, 0, 0, 0, 49563, 180075, 121543, 0, 387, 0, 128709, 1852296, 825243, 0, 0, 0, 1008, 2427, 994, 0, 49, 0, 1116, 10640, 2769, 0, 0, 0, 222, 537, 184, 0, 17, 0, 234, 1693, 351, 0, 0, 0, 2227188, 282357, 129907, 0, 25808, 0, 250935, 2673094, 1882398, 0, 0, 0, 1056, 2400, 955, 0, 140, 0, 1500, 11746, 2751 +9,2, 3, 0, 0, 0, 2, 12, 1, 0, 2, 0, 11, 110, 4, 0, 0, 0, 395, 2521, 58, 0, 294, 0, 51, 29596, 83, 0, 0, 0, 14, 75, 5, 0, 9, 0, 17, 398, 6, 0, 0, 0, 2, 12, 1, 0, 2, 0, 11, 110, 4, 0, 0, 0, 250, 3091, 19, 0, 1325, 0, 8, 104779, 40, 0, 0, 0, 13, 79, 4, 0, 17, 0, 5, 402, 5 +1,3, 1, 0, 0, 0, 285, 2544, 267, 0, 17, 0, 144, 2431, 769, 0, 0, 0, 166509, 3999480, 220998, 0, 894, 0, 222237, 2754831, 715340, 0, 0, 0, 1266, 17727, 1518, 0, 42, 0, 963, 16908, 5509, 0, 0, 0, 285, 2544, 267, 0, 17, 0, 144, 2431, 769, 0, 0, 0, 245907, 26402886, 573315, 0, 69140, 0, 351909, 8881066, 1285411, 0, 0, 0, 1371, 17829, 1683, 0, 154, 0, 1029, 18698, 6037 +9,2, 2, 0, 0, 0, 376, 170, 395, 0, 6, 0, 168, 622, 266, 0, 0, 0, 279770, 193152, 288400, 0, 80, 0, 217272, 993150, 304150, 0, 0, 0, 2724, 1254, 2669, 0, 10, 0, 1102, 3928, 1850, 0, 0, 0, 376, 170, 395, 0, 6, 0, 168, 622, 266, 0, 0, 0, 341580, 239264, 365445, 0, 4670, 0, 318672, 4140138, 334156, 0, 0, 0, 2680, 1274, 2659, 0, 37, 0, 1122, 4369, 1780 +1,3, 2, 0, 1, 0, 1028, 1073, 640, 0, 6, 0, 630, 1247, 528, 0, 10, 0, 900934, 1163142, 544494, 0, 67, 0, 1146340, 1609053, 450714, 0, 3, 0, 8050, 8815, 4518, 0, 7, 0, 4754, 9321, 3592, 0, 1, 0, 1028, 1073, 640, 0, 6, 0, 630, 1247, 528, 0, 10, 0, 2110490, 2748844, 15043658, 0, 5397, 0, 2954966, 5270283, 912562, 0, 3, 0, 8194, 8629, 4648, 0, 42, 0, 4478, 9556, 3846 +9,2, 1, 0, 0, 0, 147, 448, 192, 0, 5, 0, 171, 1009, 150, 0, 0, 0, 55560, 232286, 93648, 0, 146, 0, 277191, 1242169, 709851, 0, 0, 0, 729, 2482, 1179, 0, 17, 0, 750, 6625, 1194, 0, 0, 0, 147, 448, 192, 0, 5, 0, 171, 1009, 150, 0, 0, 0, 380490, 331084, 86334, 0, 5395, 0, 422886, 1646549, 1936110, 0, 0, 0, 825, 2496, 1167, 0, 43, 0, 1011, 7397, 1215 +2,3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10160, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21606, 2031, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 27 +10,2, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1999, 0, 0, 0, 0, 0, 4968, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 682, 0, 0, 0, 0, 0, 10224, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 60, 0 +10,2, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 540, 0, 0, 0, 0, 0, 0, 26, 20, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 14, 2, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 6, 2 +1,3, 3, 0, 0, 0, 11, 32, 8, 1, 6, 0, 12, 238, 18, 0, 0, 0, 10491, 3339, 7923, 28, 256, 0, 4981, 213922, 125115, 0, 0, 0, 81, 178, 69, 4, 25, 0, 58, 1546, 119, 0, 0, 0, 11, 32, 8, 1, 6, 0, 12, 238, 18, 0, 0, 0, 13475, 3858, 7897, 137, 6516, 0, 4569, 221064, 125909, 0, 0, 0, 82, 191, 69, 7, 52, 0, 62, 1477, 148 +9,2, 3, 0, 0, 0, 12, 15, 2, 0, 3, 0, 21, 102, 10, 0, 0, 0, 3295, 30991, 1212, 0, 1, 0, 243502, 31834, 149, 0, 0, 0, 83, 94, 18, 0, 1, 0, 105, 361, 19, 0, 0, 0, 12, 15, 2, 0, 3, 0, 21, 102, 10, 0, 0, 0, 3858, 47334, 1047, 0, 1155, 0, 382935, 39624, 787, 0, 0, 0, 82, 108, 16, 0, 19, 0, 92, 393, 35 +1,3, 1, 0, 0, 0, 39, 328, 45, 0, 4, 0, 3, 206, 9, 0, 0, 0, 7602, 127911, 8481, 0, 67, 0, 3, 161024, 16530, 0, 0, 0, 153, 2115, 231, 0, 7, 0, 3, 1493, 81, 0, 0, 0, 39, 328, 45, 0, 4, 0, 3, 206, 9, 0, 0, 0, 24198, 129653, 315660, 0, 5138, 0, 111, 296998, 14508, 0, 0, 0, 195, 1963, 255, 0, 31, 0, 15, 1532, 69 +2,3, 3, 0, 0, 0, 7, 40, 11, 0, 19, 0, 5, 178, 14, 0, 0, 0, 2102, 21018, 3293, 0, 18, 0, 4374, 83304, 66064, 0, 0, 0, 51, 260, 76, 0, 7, 0, 36, 1069, 121, 0, 0, 0, 7, 40, 11, 0, 19, 0, 5, 178, 14, 0, 0, 0, 2291, 26409, 4164, 0, 74457, 0, 7219, 888055, 210746, 0, 0, 0, 50, 269, 77, 0, 121, 0, 35, 1142, 136 +1,3, 2, 0, 0, 0, 22, 15, 18, 0, 0, 0, 46, 66, 96, 0, 0, 0, 4918, 7700, 4758, 0, 0, 0, 30472, 32276, 67470, 0, 0, 0, 112, 116, 110, 0, 0, 0, 332, 368, 678, 0, 0, 0, 22, 15, 18, 0, 0, 0, 46, 66, 96, 0, 0, 0, 18882, 8495, 19276, 0, 0, 0, 38982, 52910, 4072736, 0, 0, 0, 120, 111, 120, 0, 0, 0, 308, 442, 742 +3,3, 3, 0, 0, 0, 3, 15, 3, 3, 15, 0, 7, 108, 8, 0, 0, 0, 2477, 2961, 2553, 131, 2542, 0, 4528, 66338, 4392, 0, 0, 0, 25, 91, 25, 16, 38, 0, 56, 541, 54, 0, 0, 0, 3, 15, 3, 3, 15, 0, 7, 108, 8, 0, 0, 0, 2817, 4115, 1101, 4164, 121651, 0, 8876, 108120, 7144, 0, 0, 0, 25, 100, 24, 25, 111, 0, 64, 672, 70 +3,3, 1, 0, 1, 0, 177, 509, 180, 0, 6, 0, 156, 1882, 372, 0, 127, 0, 312204, 513233, 146253, 0, 3645, 0, 123297, 2484727, 465462, 0, 7, 0, 993, 2921, 1290, 0, 25, 0, 1140, 13724, 2928, 0, 1, 0, 177, 509, 180, 0, 6, 0, 156, 1882, 372, 0, 114, 0, 1957305, 951789, 340848, 0, 5266, 0, 603135, 4511462, 2443476, 0, 6, 0, 1137, 3132, 1431, 0, 43, 0, 1251, 14758, 3048 +2,3, 1, 2, 1, 0, 319, 795, 270, 0, 13, 0, 351, 2618, 486, 682, 230, 0, 127202, 332166, 104583, 0, 12218, 0, 366810, 2633680, 933339, 16, 7, 0, 1247, 3546, 1398, 0, 93, 0, 2640, 18351, 3813, 2, 1, 0, 319, 795, 270, 0, 13, 0, 351, 2618, 486, 72679, 314, 0, 745493, 557987, 535716, 0, 93549, 0, 1015149, 10929279, 2167419, 29, 8, 0, 1546, 3910, 1530, 0, 123, 0, 2679, 20487, 3870 +3,3, 2, 0, 0, 0, 268, 285, 426, 0, 25, 0, 466, 943, 422, 0, 0, 0, 182646, 230075, 299336, 0, 2907, 0, 509416, 3744575, 831314, 0, 0, 0, 1976, 2120, 2532, 0, 51, 0, 3826, 7213, 3324, 0, 0, 0, 268, 285, 426, 0, 25, 0, 466, 943, 422, 0, 0, 0, 2384092, 412703, 489446, 0, 18880, 0, 553734, 4762712, 1559584, 0, 0, 0, 2052, 2122, 2672, 0, 173, 0, 3556, 7775, 3632 +2,3, 2, 0, 0, 0, 1040, 472, 699, 0, 6, 0, 462, 1414, 476, 0, 0, 0, 788066, 572592, 543104, 0, 0, 0, 1318066, 2434176, 808572, 0, 0, 0, 7498, 3756, 4868, 0, 0, 0, 3550, 11210, 3530, 0, 0, 0, 1040, 472, 699, 0, 6, 0, 462, 1414, 476, 0, 0, 0, 6026130, 638352, 923173, 0, 645, 0, 4571684, 3399669, 6086808, 0, 0, 0, 7562, 3640, 4911, 0, 31, 0, 3340, 11417, 3812 +1,3, 3, 0, 0, 0, 2, 5, 0, 0, 2, 0, 0, 15, 5, 0, 0, 0, 1516, 170, 0, 0, 0, 0, 0, 5414, 6186, 0, 0, 0, 18, 25, 0, 0, 0, 0, 0, 62, 21, 0, 0, 0, 2, 5, 0, 0, 2, 0, 0, 15, 5, 0, 0, 0, 310, 485, 0, 0, 262, 0, 0, 5966, 5792, 0, 0, 0, 14, 30, 0, 0, 11, 0, 0, 94, 20 +3,3, 1, 0, 0, 0, 111, 243, 60, 0, 2, 0, 54, 1040, 126, 0, 0, 0, 24753, 49809, 53130, 0, 130, 0, 68874, 1067330, 183678, 0, 0, 0, 480, 858, 318, 0, 12, 0, 348, 7763, 852, 0, 0, 0, 111, 243, 60, 0, 2, 0, 54, 1040, 126, 0, 0, 0, 13686, 47607, 106977, 0, 7758, 0, 3605754, 2301764, 308184, 0, 0, 0, 471, 927, 372, 0, 22, 0, 465, 7987, 921 +3,3, 3, 0, 0, 0, 7, 12, 0, 0, 20, 0, 1, 80, 6, 0, 0, 0, 11507, 1972, 0, 0, 321, 0, 1458, 36717, 5944, 0, 0, 0, 61, 73, 0, 0, 37, 0, 10, 471, 48, 0, 0, 0, 7, 12, 0, 0, 20, 0, 1, 80, 6, 0, 0, 0, 21801, 2295, 0, 0, 103091, 0, 490, 103707, 5706, 0, 0, 0, 65, 78, 0, 0, 148, 0, 8, 529, 53 +3,3, 2, 0, 0, 0, 228, 191, 242, 0, 0, 0, 142, 344, 192, 0, 0, 0, 119272, 125752, 167916, 0, 0, 0, 294240, 411528, 266212, 0, 0, 0, 1642, 1412, 1804, 0, 0, 0, 1208, 2506, 1362, 0, 0, 0, 228, 191, 242, 0, 0, 0, 142, 344, 192, 0, 0, 0, 464762, 316709, 474734, 0, 0, 0, 422516, 1038902, 367764, 0, 0, 0, 1624, 1389, 1878, 0, 0, 0, 1110, 2616, 1410 +18,1, 2, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 9, 6, 0, 0, 0, 4722, 0, 0, 0, 0, 0, 8146, 17434, 6224, 0, 0, 0, 50, 0, 0, 0, 0, 0, 22, 73, 52, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 9, 6, 0, 0, 0, 18172, 0, 0, 0, 0, 0, 9900, 24838, 57016, 0, 0, 0, 54, 0, 0, 0, 0, 0, 24, 84, 70 +17,1, 2, 1, 33, 0, 2895, 1690, 1084, 7, 206, 0, 2065, 6426, 3330, 5728, 23472, 0, 3594190, 1594479, 1000078, 2, 42068, 0, 2244128, 7496064, 5150682, 12, 259, 0, 23762, 13503, 8030, 2, 569, 0, 13810, 43711, 24890, 1, 33, 0, 2895, 1690, 1084, 7, 206, 0, 2065, 6426, 3330, 5488, 819912, 0, 16989694, 50151814, 9914276, 629, 184542, 0, 8568931, 95535080, 52184368, 12, 287, 0, 23642, 14174, 8280, 37, 1478, 0, 15835, 50970, 27204 +17,1, 3, 1, 2, 0, 7, 54, 5, 0, 63, 0, 45, 218, 571, 29, 15457, 0, 2502, 19787, 430, 0, 11734, 0, 80404, 160185, 537131, 4, 18, 0, 39, 313, 24, 0, 211, 0, 290, 1245, 4518, 1, 2, 0, 7, 54, 5, 0, 63, 0, 45, 218, 571, 30873, 15448, 0, 1680, 57459, 828, 0, 872382, 0, 108147, 1079579, 191725, 14, 17, 0, 47, 372, 28, 0, 489, 0, 348, 1639, 3049 +18,1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1233, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 1278, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 36, 0 +17,1, 1, 0, 0, 0, 387, 1098, 276, 0, 52, 0, 1197, 6752, 523, 0, 0, 0, 270597, 388398, 334569, 0, 10437, 0, 1580688, 9166335, 973871, 0, 0, 0, 1575, 4881, 1956, 0, 137, 0, 8799, 49753, 3756, 0, 0, 0, 387, 1098, 276, 0, 52, 0, 1197, 6752, 523, 0, 0, 0, 524604, 1154028, 738969, 0, 98362, 0, 2614776, 31490393, 1437749, 0, 0, 0, 1785, 5391, 2019, 0, 400, 0, 9222, 53705, 3808 +19,1, 1, 0, 0, 0, 90, 171, 51, 0, 1, 0, 34, 329, 51, 0, 0, 0, 21546, 54915, 48189, 0, 149, 0, 139268, 374227, 32823, 0, 0, 0, 294, 561, 279, 0, 7, 0, 296, 2153, 300, 0, 0, 0, 90, 171, 51, 0, 1, 0, 34, 329, 51, 0, 0, 0, 20157, 78285, 78771, 0, 202, 0, 132205, 807467, 42003, 0, 0, 0, 297, 639, 291, 0, 7, 0, 278, 2135, 300 +18,1, 3, 1, 0, 0, 4, 40, 24, 0, 21, 0, 19, 213, 272, 20, 0, 0, 2064, 8333, 93227, 0, 44, 0, 24033, 192966, 237332, 4, 0, 0, 24, 248, 205, 0, 16, 0, 120, 1209, 1536, 1, 0, 0, 4, 40, 24, 0, 21, 0, 19, 213, 272, 84, 0, 0, 4713, 13156, 86628, 0, 43374, 0, 26787, 344756, 536005, 6, 0, 0, 29, 272, 200, 0, 142, 0, 150, 1368, 1272 +18,1, 2, 0, 6, 0, 1044, 900, 764, 0, 59, 0, 1336, 4574, 1734, 0, 3492, 0, 877906, 865206, 396762, 0, 10001, 0, 2498450, 5266335, 1874806, 0, 44, 0, 6830, 5582, 3680, 0, 162, 0, 9876, 30991, 12288, 0, 6, 0, 1044, 900, 764, 0, 59, 0, 1336, 4574, 1734, 0, 2497, 0, 2658344, 2392927, 4481930, 0, 199532, 0, 6732914, 12754035, 9436990, 0, 39, 0, 6660, 5805, 3840, 0, 413, 0, 10638, 34189, 13230 +17,1, 2, 0, 0, 0, 12, 2, 6, 0, 0, 0, 2, 23, 6, 0, 0, 0, 1940, 154, 888, 0, 0, 0, 186, 367566, 2182, 0, 0, 0, 54, 12, 26, 0, 0, 0, 12, 174, 36, 0, 0, 0, 12, 2, 6, 0, 0, 0, 2, 23, 6, 0, 0, 0, 1514, 44, 2100, 0, 0, 0, 190, 372927, 155306, 0, 0, 0, 48, 8, 26, 0, 0, 0, 12, 189, 54 +19,1, 3, 0, 0, 0, 0, 5, 1, 0, 5, 0, 2, 36, 70, 0, 0, 0, 0, 331, 193, 0, 9, 0, 233, 23019, 113874, 0, 0, 0, 0, 26, 7, 0, 4, 0, 12, 237, 553, 0, 0, 0, 0, 5, 1, 0, 5, 0, 2, 36, 70, 0, 0, 0, 0, 562, 70, 0, 50538, 0, 82, 28254, 97228, 0, 0, 0, 0, 31, 6, 0, 36, 0, 9, 239, 470 +17,1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2581, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0 +19,1, 2, 0, 0, 0, 240, 114, 176, 0, 3, 0, 97, 407, 192, 0, 0, 0, 169070, 148108, 104078, 0, 0, 0, 120860, 418848, 328976, 0, 0, 0, 1736, 930, 1092, 0, 0, 0, 652, 2888, 1382, 0, 0, 0, 240, 114, 176, 0, 3, 0, 97, 407, 192, 0, 0, 0, 220746, 164078, 361418, 0, 268, 0, 135960, 678904, 618056, 0, 0, 0, 1756, 930, 1114, 0, 16, 0, 616, 2952, 1450 +18,1, 1, 0, 0, 0, 405, 882, 249, 0, 11, 0, 216, 2669, 486, 0, 0, 0, 97314, 375873, 175098, 0, 64, 0, 99753, 2735555, 496263, 0, 0, 0, 1335, 3375, 1506, 0, 18, 0, 927, 19507, 3270, 0, 0, 0, 405, 882, 249, 0, 11, 0, 216, 2669, 486, 0, 0, 0, 150363, 597807, 262425, 0, 39103, 0, 1076358, 4165383, 741708, 0, 0, 0, 1509, 3765, 1500, 0, 87, 0, 1386, 20520, 3417 +17,1, 1, 0, 0, 0, 9, 21, 4, 0, 2, 0, 15, 58, 15, 0, 0, 0, 0, 2196, 1448, 0, 0, 0, 22092, 16080, 14187, 0, 0, 0, 0, 69, 28, 0, 0, 0, 120, 375, 114, 0, 0, 0, 9, 21, 4, 0, 2, 0, 15, 58, 15, 0, 0, 0, 15, 1302, 353, 0, 202, 0, 32706, 61646, 16137, 0, 0, 0, 9, 72, 25, 0, 12, 0, 126, 399, 120 +19,1, 1, 0, 0, 0, 42, 96, 12, 0, 4, 0, 18, 134, 25, 0, 0, 0, 915, 6561, 6015, 0, 0, 0, 2778, 50907, 29977, 0, 0, 0, 51, 186, 39, 0, 0, 0, 78, 633, 219, 0, 0, 0, 42, 96, 12, 0, 4, 0, 18, 134, 25, 0, 0, 0, 1194, 18456, 3411, 0, 216, 0, 1257, 76899, 89522, 0, 0, 0, 78, 255, 39, 0, 18, 0, 93, 798, 233 +21,1, 2, 0, 2, 0, 191, 84, 118, 0, 14, 0, 200, 1382, 308, 0, 663, 0, 91018, 96047, 43644, 0, 2359, 0, 211912, 1997955, 253232, 0, 15, 0, 1186, 595, 728, 0, 32, 0, 1292, 9684, 2100, 0, 2, 0, 191, 84, 118, 0, 14, 0, 200, 1382, 308, 0, 594, 0, 122392, 179188, 522350, 0, 13779, 0, 316474, 8449675, 569560, 0, 15, 0, 1172, 609, 744, 0, 94, 0, 1490, 11248, 2296 +20,1, 1, 0, 1, 0, 282, 656, 228, 0, 58, 0, 514, 5186, 1464, 0, 368, 0, 200070, 250996, 153975, 0, 69779, 0, 603928, 6866587, 1744866, 0, 8, 0, 1431, 3352, 1545, 0, 198, 0, 3166, 36270, 9969, 0, 1, 0, 282, 656, 228, 0, 58, 0, 514, 5186, 1464, 0, 62, 0, 3473535, 453616, 209400, 0, 271711, 0, 1658197, 15820403, 7174386, 0, 5, 0, 1563, 3643, 1587, 0, 493, 0, 3900, 40823, 11193 +21,1, 1, 0, 0, 0, 96, 316, 60, 0, 11, 0, 90, 1615, 231, 0, 0, 0, 31416, 88356, 22074, 0, 10985, 0, 41217, 1754221, 363048, 0, 0, 0, 456, 1692, 342, 0, 60, 0, 564, 10608, 1731, 0, 0, 0, 96, 316, 60, 0, 11, 0, 90, 1615, 231, 0, 0, 0, 52212, 2229561, 17091, 0, 58972, 0, 373542, 3467570, 667014, 0, 0, 0, 513, 1740, 306, 0, 85, 0, 804, 11957, 1947 +19,1, 3, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 40, 13, 0, 0, 0, 0, 293, 0, 0, 318, 0, 0, 13846, 38149, 0, 0, 0, 0, 27, 0, 0, 10, 0, 0, 188, 98, 0, 0, 0, 0, 5, 0, 0, 5, 0, 0, 40, 13, 0, 0, 0, 0, 459, 0, 0, 18783, 0, 0, 109242, 38838, 0, 0, 0, 0, 30, 0, 0, 33, 0, 0, 270, 80 +20,1, 3, 0, 0, 0, 1, 215, 6, 4, 13, 0, 18, 197, 31, 0, 0, 0, 211, 3522, 2473, 3251, 3036, 0, 20423, 156434, 11721, 0, 0, 0, 7, 385, 39, 26, 51, 0, 101, 1078, 136, 0, 0, 0, 1, 215, 6, 4, 13, 0, 18, 197, 31, 0, 0, 0, 214, 4236, 9836, 8590, 20052, 0, 86957, 203025, 14512, 0, 0, 0, 7, 487, 47, 39, 112, 0, 95, 1391, 164 +19,1, 2, 0, 0, 0, 96, 26, 56, 0, 3, 0, 310, 168, 150, 0, 0, 0, 51964, 17976, 19116, 0, 73, 0, 118662, 116510, 100912, 0, 0, 0, 622, 206, 280, 0, 6, 0, 2406, 1217, 932, 0, 0, 0, 96, 26, 56, 0, 3, 0, 310, 168, 150, 0, 0, 0, 96536, 31070, 15584, 0, 4052, 0, 369170, 147431, 128668, 0, 0, 0, 638, 210, 278, 0, 20, 0, 2584, 1234, 886 +21,1, 3, 0, 1, 0, 3, 5, 1, 1, 7, 0, 0, 50, 11, 0, 0, 0, 615, 4189, 211, 6, 2116, 0, 0, 52855, 5935, 0, 0, 0, 20, 37, 7, 2, 36, 0, 0, 340, 67, 0, 1, 0, 3, 5, 1, 1, 7, 0, 0, 50, 11, 0, 11, 0, 493, 3134, 302, 189, 60424, 0, 0, 95847, 5324, 0, 3, 0, 19, 38, 8, 7, 56, 0, 0, 371, 62 +20,1, 2, 0, 0, 0, 604, 326, 466, 0, 18, 0, 482, 1331, 694, 0, 0, 0, 517494, 213050, 338524, 0, 6751, 0, 793322, 1667213, 849880, 0, 0, 0, 4512, 2450, 3448, 0, 45, 0, 3330, 8963, 5048, 0, 0, 0, 604, 326, 466, 0, 18, 0, 482, 1331, 694, 0, 0, 0, 1380158, 440924, 1387112, 0, 49001, 0, 1231546, 5942380, 2567424, 0, 0, 0, 4314, 2410, 3572, 0, 140, 0, 3700, 10049, 5408 +22,1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0 +21,1, 2, 1, 1, 0, 985, 1075, 888, 1, 66, 0, 691, 7477, 1248, 0, 1035, 0, 626278, 533283, 400630, 668, 16348, 0, 1260240, 7428762, 1100050, 0, 10, 0, 7286, 7654, 6106, 9, 237, 0, 5249, 45936, 8638, 1, 1, 0, 985, 1075, 888, 1, 66, 0, 691, 7477, 1248, 81, 457, 0, 1445425, 5950009, 1390202, 1364, 1565174, 0, 4331936, 29512542, 4595126, 6, 8, 0, 7252, 7312, 6022, 10, 527, 0, 5470, 45061, 9196 +22,1, 1, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 705, 1455, 0, 0, 0, 0, 0, 19822, 30, 0, 0, 0, 21, 24, 0, 0, 0, 0, 0, 139, 9, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 19, 3, 0, 0, 0, 6723, 39471, 0, 0, 0, 0, 0, 43608, 231, 0, 0, 0, 33, 39, 0, 0, 0, 0, 0, 152, 18 +21,1, 1, 4, 23, 0, 689, 2309, 657, 0, 156, 0, 423, 7671, 1416, 2743, 94466, 0, 437741, 1564876, 738543, 0, 79763, 0, 508077, 8355160, 1719474, 35, 207, 0, 4123, -2147472894, 3111, 0, 722, 0, 3123, 53215, 10383, 4, 23, 0, 689, 2309, 657, 0, 156, 0, 423, 7671, 1416, 16939, 514178, 0, 1251389, 4900242, 1257939, 0, 2536566, 0, 1053069, 39916317, 13061787, 43, 253, 0, 4532, -2147472069, 3453, 0, 1216, 0, 3480, 61031, 11115 +21,1, 3, 0, 1, 0, 0, 533, 1, 1, 21, 0, 14, 479, 37, 0, 440, 0, 0, 17494, 47, 0, 3020, 0, 162800, 686760, 48759, 0, 8, 0, 0, 1085, 5, 0, 71, 0, 108, 3366, 283, 0, 1, 0, 0, 533, 1, 1, 21, 0, 14, 479, 37, 0, 609, 0, 0, 98300, 7, 11, 120457, 0, 157982, 1104971, 59726, 0, 9, 0, 0, 1287, 3, 3, 202, 0, 117, 3682, 291 +22,1, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 1974, 0, 0, 0, 14832, 28, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 42, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 904, 0, 0, 0, 12204, 20, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 38, 6, 0 +22,1, 3, 0, 0, 0, 3, 19, 0, 2, 22, 0, 8, 77, 5, 0, 0, 0, 1599, 7722, 0, 288, 3671, 0, 240, 54315, 2102, 0, 0, 0, 23, 128, 0, 14, 74, 0, 21, 450, 24, 0, 0, 0, 3, 19, 0, 2, 22, 0, 8, 77, 5, 0, 0, 0, 642, 7170, 0, 2470, 135827, 0, 778, 69411, 1563, 0, 0, 0, 19, 126, 0, 20, 177, 0, 28, 532, 34 +21,1, 2, 0, 0, 0, 274, 142, 240, 0, 20, 0, 166, 1459, 224, 0, 0, 0, 219440, 149400, 235650, 0, 694, 0, 144310, 1909209, 499564, 0, 0, 0, 1892, 1136, 1632, 0, 41, 0, 1244, 10282, 1464, 0, 0, 0, 274, 142, 240, 0, 20, 0, 166, 1459, 224, 0, 0, 0, 441706, 254670, 3703660, 0, 116280, 0, 343514, 10468188, 761664, 0, 0, 0, 1878, 1148, 1666, 0, 148, 0, 1286, 11111, 1838 +22,1, 1, 2, 25, 0, 379, 1304, 957, 0, 112, 0, 591, 7620, 1500, 2710, 12573, 0, 164528, 828276, 247605, 0, 31440, 0, 1221738, 9375948, 1894692, 19, 157, 0, 1919, 7172, 3429, 0, 408, 0, 4317, 51454, 10581, 2, 25, 0, 379, 1304, 957, 0, 112, 0, 591, 7620, 1500, 978, 124896, 0, 367575, 2293602, 425547, 0, 1353552, 0, 6340149, 25506304, 9888576, 15, 230, 0, 2187, 8050, 3927, 0, 916, 0, 5007, 61765, 12072 +21,1, 1, 0, 4, 0, 159, 830, 108, 0, 42, 0, 108, 3967, 387, 0, 1244, 0, 61059, 345481, 81207, 0, 16336, 0, 109152, 5428784, 533892, 0, 30, 0, 507, 5076, 624, 0, 131, 0, 840, 27675, 3003, 0, 4, 0, 159, 830, 108, 0, 42, 0, 108, 3967, 387, 0, 3321, 0, 74664, 704442, 207777, 0, 137254, 0, 201669, 31376659, 794424, 0, 31, 0, 558, 4952, 669, 0, 312, 0, 861, 31215, 3084 +21,1, 3, 0, 0, 0, 1, 10, 2, 0, 8, 0, 2, 66, 20, 0, 0, 0, 2, 884, 474, 0, 15, 0, 7078, 59737, 4594, 0, 0, 0, 1, 56, 10, 0, 6, 0, 16, 390, 69, 0, 0, 0, 1, 10, 2, 0, 8, 0, 2, 66, 20, 0, 0, 0, 40, 1950, 698, 0, 69344, 0, 6531, 99131, 4358, 0, 0, 0, 5, 65, 16, 0, 59, 0, 14, 470, 96 +22,1, 2, 0, 0, 0, 443, 318, 350, 0, 34, 0, 490, 2502, 690, 0, 0, 0, 489825, 219116, 406486, 0, 10383, 0, 421640, 3583969, 616558, 0, 0, 0, 3250, 2388, 2492, 0, 108, 0, 3648, 17166, 4854, 0, 0, 0, 443, 318, 350, 0, 34, 0, 490, 2502, 690, 0, 0, 0, 1987726, 581868, 875074, 0, 29998, 0, 3349708, 15740438, 2841748, 0, 0, 0, 3301, 2408, 2542, 0, 244, 0, 3768, 19356, 5354 +23,1, 3, 0, 0, 0, 1, 10, 2, 0, 3, 0, 4, 42, 7, 0, 0, 0, 564, 3223, 5891, 0, 21, 0, 4873, 16800, 739, 0, 0, 0, 9, 66, 18, 0, 6, 0, 34, 195, 39, 0, 0, 0, 1, 10, 2, 0, 3, 0, 4, 42, 7, 0, 0, 0, 245, 3788, 5780, 0, 751, 0, 5518, 30184, 633, 0, 0, 0, 7, 70, 17, 0, 19, 0, 38, 255, 41 +22,1, 3, 0, 0, 0, 6, 22, 0, 0, 11, 0, 2, 87, 14, 0, 0, 0, 4425, 12244, 0, 0, 338, 0, 2371, 45365, 113907, 0, 0, 0, 50, 157, 0, 0, 29, 0, 18, 433, 51, 0, 0, 0, 6, 22, 0, 0, 11, 0, 2, 87, 14, 0, 0, 0, 14914, 14594, 0, 0, 52736, 0, 3037, 75641, 111294, 0, 0, 0, 52, 160, 0, 0, 78, 0, 20, 489, 39 +22,1, 1, 2, 2, 0, 409, 1435, 234, 0, 110, 0, 303, 6956, 1029, 482, 228, 0, 151477, 627699, 288675, 0, 39216, 0, 631026, 8062980, 1148955, 15, 12, 0, 2115, 5775, 1473, 0, 399, 0, 2193, 45960, 7296, 2, 2, 0, 409, 1435, 234, 0, 110, 0, 303, 6956, 1029, 3300, 176, 0, 338469, 3876214, 843000, 0, 283372, 0, 4984080, 29346635, 7707744, 18, 10, 0, 2256, 6422, 1578, 0, 903, 0, 2406, 54468, 7872 +23,1, 1, 0, 19, 0, 288, 722, 84, 0, 60, 0, 147, 3184, 507, 0, 14768, 0, 136416, 450499, 39165, 0, 5317, 0, 120429, 3141934, 691257, 0, 163, 0, 1572, 3569, 519, 0, 154, 0, 963, 21197, 3699, 0, 19, 0, 288, 722, 84, 0, 60, 0, 147, 3184, 507, 0, 94072, 0, 232380, 752369, 71298, 0, 141753, 0, 781683, 19913521, 12252231, 0, 181, 0, 1533, 3833, 528, 0, 467, 0, 1221, 25306, 4110 +23,1, 2, 0, 1, 0, 348, 217, 330, 0, 13, 0, 131, 651, 206, 0, 736, 0, 210994, 86812, 248414, 0, 289, 0, 43011, 899787, 169228, 0, 9, 0, 2314, 1515, 1746, 0, 17, 0, 628, 4699, 1464, 0, 1, 0, 348, 217, 330, 0, 13, 0, 131, 651, 206, 0, 10753, 0, 342184, 408909, 908344, 0, 59079, 0, 191791, 1659335, 365800, 0, 13, 0, 2312, 1601, 1804, 0, 89, 0, 854, 5099, 1556 +22,1, 2, 0, 1, 0, 948, 273, 426, 0, 33, 0, 398, 2222, 608, 0, 65, 0, 350942, 304629, 273604, 0, 8826, 0, 325580, 4455799, 974180, 0, 6, 0, 4290, 2072, 2934, 0, 89, 0, 2628, 16198, 4238, 0, 1, 0, 948, 273, 426, 0, 33, 0, 398, 2222, 608, 0, 3374, 0, 1030858, 452476, 475324, 0, 167489, 0, 863356, 8034394, 4998664, 0, 11, 0, 4618, 2161, 2994, 0, 261, 0, 2496, 17355, 4364 diff --git a/2014-Group5/Matlab/load_3_stocks.m b/2014-Group5/Matlab/load_3_stocks.m new file mode 100644 index 0000000..245eb30 --- /dev/null +++ b/2014-Group5/Matlab/load_3_stocks.m @@ -0,0 +1,297 @@ +% Information Retrieval and Data Mining +% University College London 2013/2014 +% Coursework 2 + +% Daily closing price The final price at which a security is traded on a given trading day +% Daily traded volume The volume which is traded on a given trading day +% Daily price change The change in the price of a stock from the previous trading day's close to the current day's close +% Daily absolute price change The norm for computing asset performance change + +% Features: Y --> pos1: volume traded, pos2: closing price, pos3: daily change price +% pos 4: abs daily change price + +function [Yibm,Yintel,Yge]=load_3_stocks() + +%ibm=csvread('stock-prices/IBM.csv',1,0); % skip first row +A = importdata('stock-prices/IBM.csv',',',1); +ibm=A.data; +ibm=flipud(ibm); +A = importdata('stock-prices/GE.csv',',',1); +ge=A.data; +ge=flipud(ge); +A = importdata('stock-prices/intel.csv',',',1); +intel=A.data; +intel=flipud(intel); + +% pos1: Open, pos2: High, pos3: Low, pos4: Close, pos5: Volume +% pos6: Adj Close + +Yibm=zeros(50,1); +Yintel=zeros(50,1); +Yge=zeros(50,1); + +% Days: +% 13/01, 14/01, 15/01, 16/01, 17/01, 18/01*, 19/01*, 20/01*, +% 21/01, 22/01, 23/01, 24/01, 25/01*, 26/01*, 27/01, 28/01, 29/01, 30/01, +% 31/01 +% Total days Jan: 19 (5 fake) +% 1/02*, 2/02*, 3/02, 4/02, 5/02, 6/02, 7/02, 8/02*, 9/02*, 10/02, 11/02, +% 12/02, 13/02, 14/02, 15/02*, 16/02*, 17/02*, 18/02, 19/02, 20/02, 21/02, +% 22/02*, 23/02*, 24/02, 25/02, 26/02, 27/02, 28/02 +% Total days Feb: 28 (9 fake) +% 1/02*, 2/03*, 3/03 +% Total days Mar: 3 (2 fake) +% -- +% Total days: 50 (16 fake) + +% 5-3*-4-2*-5-2*-5-2*-5-3*-4-2*-5-2*-1 + + +% ibm on 10/01: 18,770.00 18,770.00 18,638.00 18,687.00 300 18,584.61 +% intel on 10/01: 25.50 25.85 25.50 25.53 30,588,800 25.29 +% ge on 10/01: 27.19 27.23 26.86 26.96 38,828,600 26.73 + + +%% IBM + +% 5 +Yibm(1:5,1)=ibm(1:5,5); +Yibm(1:5,2)=ibm(1:5,4); +Yibm(1,3)=ibm(1,4)-18687; +Yibm(2:5,3)=ibm(2:5,4)-ibm(1:4,4); +Yibm(1:5,4)=abs(Yibm(1:5,3)); +% 3* +Yibm(6:8,1)=repmat(ibm(5,5),3,1); +Yibm(6:8,2)=repmat(ibm(5,4),3,1); +Yibm(6:8,3)=repmat(Yibm(5,3),3,1); +Yibm(6:8,4)=abs(Yibm(6:8,3)); +% 4 +Yibm(9:12,1)=ibm(6:9,5); +Yibm(9:12,2)=ibm(6:9,4); +Yibm(9:12,3)=ibm(6:9,4)-ibm(5:8,4); +Yibm(9:12,4)=abs(Yibm(9:12,3)); +% 2* +Yibm(13:14,1)=repmat(ibm(9,5),2,1); +Yibm(13:14,2)=repmat(ibm(9,4),2,1); +Yibm(13:14,3)=repmat(Yibm(12,3),2,1); +Yibm(13:14,4)=abs(Yibm(13:14,3)); +% 5 +Yibm(15:19,1)=ibm(10:14,5); +Yibm(15:19,2)=ibm(10:14,4); +Yibm(15:19,3)=ibm(10:14,4)-ibm(9:13,4); +Yibm(15:19,4)=abs(Yibm(15:19,3)); +% 2* +Yibm(20:21,1)=repmat(ibm(14,5),2,1); +Yibm(20:21,2)=repmat(ibm(14,4),2,1); +Yibm(20:21,3)=repmat(Yibm(19,3),2,1); +Yibm(20:21,4)=abs(Yibm(20:21,3)); +% 5 +Yibm(22:26,1)=ibm(15:19,5); +Yibm(22:26,2)=ibm(15:19,4); +Yibm(22:26,3)=ibm(15:19,4)-ibm(14:18,4); +Yibm(22:26,4)=abs(Yibm(15:19,3)); +% 2* +Yibm(27:28,1)=repmat(ibm(19,5),2,1); +Yibm(27:28,2)=repmat(ibm(19,4),2,1); +Yibm(27:28,3)=repmat(Yibm(26,3),2,1); +Yibm(27:28,4)=abs(Yibm(27:28,3)); +% 5 +Yibm(29:33,1)=ibm(20:24,5); +Yibm(29:33,2)=ibm(20:24,4); +Yibm(29:33,3)=ibm(20:24,4)-ibm(19:23,4); +Yibm(29:33,4)=abs(Yibm(20:24,3)); +% 3* +Yibm(34:36,1)=repmat(ibm(24,5),3,1); +Yibm(34:36,2)=repmat(ibm(24,4),3,1); +Yibm(34:36,3)=repmat(Yibm(33,3),3,1); +Yibm(34:36,4)=abs(Yibm(34:36,3)); +% 4 +Yibm(37:40,1)=ibm(25:28,5); +Yibm(37:40,2)=ibm(25:28,4); +Yibm(37:40,3)=ibm(25:28,4)-ibm(24:27,4); +Yibm(37:40,4)=abs(Yibm(25:28,3)); +% 2* +Yibm(41:42,1)=repmat(ibm(28,5),2,1); +Yibm(41:42,2)=repmat(ibm(28,4),2,1); +Yibm(41:42,3)=repmat(Yibm(40,3),2,1); +Yibm(41:42,4)=abs(Yibm(41:42,3)); +% 5 +Yibm(43:47,1)=ibm(29:33,5); +Yibm(43:47,2)=ibm(29:33,4); +Yibm(43:47,3)=ibm(29:33,4)-ibm(28:32,4); +Yibm(43:47,4)=abs(Yibm(29:33,3)); +% 2* +Yibm(48:49,1)=repmat(ibm(33,5),2,1); +Yibm(48:49,2)=repmat(ibm(33,4),2,1); +Yibm(48:49,3)=repmat(Yibm(47,3),2,1); +Yibm(48:49,4)=abs(Yibm(48:49,3)); +% 1 +Yibm(50,1)=ibm(34,5); +Yibm(50,2)=ibm(34,4); +Yibm(50,3)=ibm(34,4)-ibm(33,4); +Yibm(50,4)=abs(Yibm(34,3)); + + +%% GE + +% 5 +Yge(1:5,1)=ge(1:5,5); +Yge(1:5,2)=ge(1:5,4); +Yge(1,3)=ge(1,4)-26.96; +Yge(2:5,3)=ge(2:5,4)-ge(1:4,4); +Yge(1:5,4)=abs(Yge(1:5,3)); +% 3* +Yge(6:8,1)=repmat(ge(5,5),3,1); +Yge(6:8,2)=repmat(ge(5,4),3,1); +Yge(6:8,3)=repmat(Yge(5,3),3,1); +Yge(6:8,4)=abs(Yge(6:8,3)); +% 4 +Yge(9:12,1)=ge(6:9,5); +Yge(9:12,2)=ge(6:9,4); +Yge(9:12,3)=ge(6:9,4)-ge(5:8,4); +Yge(9:12,4)=abs(Yge(9:12,3)); +% 2* +Yge(13:14,1)=repmat(ge(9,5),2,1); +Yge(13:14,2)=repmat(ge(9,4),2,1); +Yge(13:14,3)=repmat(Yge(12,3),2,1); +Yge(13:14,4)=abs(Yge(13:14,3)); +% 5 +Yge(15:19,1)=ge(10:14,5); +Yge(15:19,2)=ge(10:14,4); +Yge(15:19,3)=ge(10:14,4)-ge(9:13,4); +Yge(15:19,4)=abs(Yge(15:19,3)); +% 2* +Yge(20:21,1)=repmat(ge(14,5),2,1); +Yge(20:21,2)=repmat(ge(14,4),2,1); +Yge(20:21,3)=repmat(Yge(19,3),2,1); +Yge(20:21,4)=abs(Yge(20:21,3)); +% 5 +Yge(22:26,1)=ge(15:19,5); +Yge(22:26,2)=ge(15:19,4); +Yge(22:26,3)=ge(15:19,4)-ge(14:18,4); +Yge(22:26,4)=abs(Yge(15:19,3)); +% 2* +Yge(27:28,1)=repmat(ge(19,5),2,1); +Yge(27:28,2)=repmat(ge(19,4),2,1); +Yge(27:28,3)=repmat(Yge(26,3),2,1); +Yge(27:28,4)=abs(Yge(27:28,3)); +% 5 +Yge(29:33,1)=ge(20:24,5); +Yge(29:33,2)=ge(20:24,4); +Yge(29:33,3)=ge(20:24,4)-ge(19:23,4); +Yge(29:33,4)=abs(Yge(20:24,3)); +% 3* +Yge(34:36,1)=repmat(ge(24,5),3,1); +Yge(34:36,2)=repmat(ge(24,4),3,1); +Yge(34:36,3)=repmat(Yge(33,3),3,1); +Yge(34:36,4)=abs(Yge(34:36,3)); +% 4 +Yge(37:40,1)=ge(25:28,5); +Yge(37:40,2)=ge(25:28,4); +Yge(37:40,3)=ge(25:28,4)-ge(24:27,4); +Yge(37:40,4)=abs(Yge(25:28,3)); +% 2* +Yge(41:42,1)=repmat(ge(28,5),2,1); +Yge(41:42,2)=repmat(ge(28,4),2,1); +Yge(41:42,3)=repmat(Yge(40,3),2,1); +Yge(41:42,4)=abs(Yge(41:42,3)); +% 5 +Yge(43:47,1)=ge(29:33,5); +Yge(43:47,2)=ge(29:33,4); +Yge(43:47,3)=ge(29:33,4)-ge(28:32,4); +Yge(43:47,4)=abs(Yge(29:33,3)); +% 2* +Yge(48:49,1)=repmat(ge(33,5),2,1); +Yge(48:49,2)=repmat(ge(33,4),2,1); +Yge(48:49,3)=repmat(Yge(47,3),2,1); +Yge(48:49,4)=abs(Yge(48:49,3)); +% 1 +Yge(50,1)=ge(34,5); +Yge(50,2)=ge(34,4); +Yge(50,3)=ge(34,4)-ge(33,4); +Yge(50,4)=abs(Yge(34,3)); + +%% Intel + +% 5 +Yintel(1:5,1)=intel(1:5,5); +Yintel(1:5,2)=intel(1:5,4); +Yintel(1,3)=intel(1,4)-25.53; +Yintel(2:5,3)=intel(2:5,4)-intel(1:4,4); +Yintel(1:5,4)=abs(Yintel(1:5,3)); +% 3* +Yintel(6:8,1)=repmat(intel(5,5),3,1); +Yintel(6:8,2)=repmat(intel(5,4),3,1); +Yintel(6:8,3)=repmat(Yintel(5,3),3,1); +Yintel(6:8,4)=abs(Yintel(6:8,3)); +% 4 +Yintel(9:12,1)=intel(6:9,5); +Yintel(9:12,2)=intel(6:9,4); +Yintel(9:12,3)=intel(6:9,4)-intel(5:8,4); +Yintel(9:12,4)=abs(Yintel(9:12,3)); +% 2* +Yintel(13:14,1)=repmat(intel(9,5),2,1); +Yintel(13:14,2)=repmat(intel(9,4),2,1); +Yintel(13:14,3)=repmat(Yintel(12,3),2,1); +Yintel(13:14,4)=abs(Yintel(13:14,3)); +% 5 +Yintel(15:19,1)=intel(10:14,5); +Yintel(15:19,2)=intel(10:14,4); +Yintel(15:19,3)=intel(10:14,4)-intel(9:13,4); +Yintel(15:19,4)=abs(Yintel(15:19,3)); +% 2* +Yintel(20:21,1)=repmat(intel(14,5),2,1); +Yintel(20:21,2)=repmat(intel(14,4),2,1); +Yintel(20:21,3)=repmat(Yintel(19,3),2,1); +Yintel(20:21,4)=abs(Yintel(20:21,3)); +% 5 +Yintel(22:26,1)=intel(15:19,5); +Yintel(22:26,2)=intel(15:19,4); +Yintel(22:26,3)=intel(15:19,4)-intel(14:18,4); +Yintel(22:26,4)=abs(Yintel(15:19,3)); +% 2* +Yintel(27:28,1)=repmat(intel(19,5),2,1); +Yintel(27:28,2)=repmat(intel(19,4),2,1); +Yintel(27:28,3)=repmat(Yintel(26,3),2,1); +Yintel(27:28,4)=abs(Yintel(27:28,3)); +% 5 +Yintel(29:33,1)=intel(20:24,5); +Yintel(29:33,2)=intel(20:24,4); +Yintel(29:33,3)=intel(20:24,4)-intel(19:23,4); +Yintel(29:33,4)=abs(Yintel(20:24,3)); +% 3* +Yintel(34:36,1)=repmat(intel(24,5),3,1); +Yintel(34:36,2)=repmat(intel(24,4),3,1); +Yintel(34:36,3)=repmat(Yintel(33,3),3,1); +Yintel(34:36,4)=abs(Yintel(34:36,3)); +% 4 +Yintel(37:40,1)=intel(25:28,5); +Yintel(37:40,2)=intel(25:28,4); +Yintel(37:40,3)=intel(25:28,4)-intel(24:27,4); +Yintel(37:40,4)=abs(Yintel(25:28,3)); +% 2* +Yintel(41:42,1)=repmat(intel(28,5),2,1); +Yintel(41:42,2)=repmat(intel(28,4),2,1); +Yintel(41:42,3)=repmat(Yintel(40,3),2,1); +Yintel(41:42,4)=abs(Yintel(41:42,3)); +% 5 +Yintel(43:47,1)=intel(29:33,5); +Yintel(43:47,2)=intel(29:33,4); +Yintel(43:47,3)=intel(29:33,4)-intel(28:32,4); +Yintel(43:47,4)=abs(Yintel(29:33,3)); +% 2* +Yintel(48:49,1)=repmat(intel(33,5),2,1); +Yintel(48:49,2)=repmat(intel(33,4),2,1); +Yintel(48:49,3)=repmat(Yintel(47,3),2,1); +Yintel(48:49,4)=abs(Yintel(48:49,3)); +% 1 +Yintel(50,1)=intel(34,5); +Yintel(50,2)=intel(34,4); +Yintel(50,3)=intel(34,4)-intel(33,4); +Yintel(50,4)=abs(Yintel(34,3)); + +pi; + + + + diff --git a/2014-Group5/authors.txt b/2014-Group5/authors.txt new file mode 100644 index 0000000..133919f --- /dev/null +++ b/2014-Group5/authors.txt @@ -0,0 +1,5 @@ +Authors: + +Iulian Serban +David Sierra Gonzalez +Xuyang Wu diff --git a/2014-Group5/java-readme.txt b/2014-Group5/java-readme.txt new file mode 100644 index 0000000..22e3eec --- /dev/null +++ b/2014-Group5/java-readme.txt @@ -0,0 +1,13 @@ +The src document applies the source codes for the coursework such as analysis the raw twitter data and the sentiment analysis platforms, Stanford and Lingpipe. + +-classifier.txt +This file contains the sentiment classifier collection which will be used to do the Lingpipe sentiment process. + +-SentimentClassfier.java +This java source code is used to do the Lingpipe sentiment process and get the result by positive, neutral and negative. + +-testlingpipe.java +This java source file used to find the each companies data, user friends, followers and sentiment analysis for each twitter, then summary and create the results file. The sentiment analysis process is using Lingpipe. + +-test +This java source file is used the Stanford¡¯s Deeply Moving platform for sentiment analysis to testing the data to compare with the Lingpipe results. diff --git a/2014-Group5/matlab-readme.txt b/2014-Group5/matlab-readme.txt new file mode 100644 index 0000000..e9fb0bf --- /dev/null +++ b/2014-Group5/matlab-readme.txt @@ -0,0 +1,60 @@ +The src document applies the source codes for the coursework such as analysis the raw twitter data and the sentiment analysis platforms, Stanford and Lingpipe. + + +- BasicLassoModel.m +Code for the linear regression model based on tweet counts only and stock features with LASSO regularization. + +- BasicLinearModel.m +Code for the linear regression model based on tweet counts only and stock features with no regularization. + +- commands.txt +Misc commands used to edit raw Twitter data. + +- complete.csv +72 extracted Twitter features. + +- ConstantModel.m +Code for the Constant Model. + +- ExtrapolateStockData.m +Uses an AR(2) model to extrapolate the stock data over weekends and holidays. This gives us a total of 50 training days. + +- FeatureAnalysis.m +Performs the feature analysis described in the report. + +- LinearLasso.m +Performs a linear regression with LASSO regularization on arbitrary input data. + +- LinearLasso.m +Performs a linear regression with LASSO regularization on arbitrary input data. + +- LinearRR.m +Performs a linear ridge regression on arbitrary input data. We set the alpha parameter strictly positive but close to zero to achieve unique solution. + +- Load_3_stocks.m +Initial code not used anymore. + +- PolynomialFeatureMap.m +Polynomial feature function which transforms (X,Y) input into (X,Y, X_1 Y_1, X_2 Y_2 etc.) inputs. + +- PolynomialFeaturesLassoModel.m +Code for the linear regression model based on PPCA features and stock features with LASSO regularization. + +- PolynomialFeaturesLinearModel.m +Code for the linear regression model based on PPCA features and stock features with no regularization. + +- ProcessTwitterCSVFile.m +Converts Twitter features into Matlab matrices for further processing. + +- RandomSimulations.m +Simulates the N(0,1) Random Model and gives its MAE, MSE and Accuracy + +- SelectedFeaturesPlot.m +Plots the four selected features as discussed in the report. + +- SentimentLassoModel.m +Code for the linear regression model based on four selected Twitter features and stock features with LASSO regularization. + +- SentimentLinearModel.m +Code for the linear regression model based on four selected Twitter features and stock features with no regularization. + diff --git a/2014-Group5/report.pdf b/2014-Group5/report.pdf new file mode 100644 index 0000000..f9d3daa Binary files /dev/null and b/2014-Group5/report.pdf differ