From 43c9f96035d172045c62f7a67faa6e07aa55d355 Mon Sep 17 00:00:00 2001 From: dji Date: Tue, 28 Feb 2023 16:10:30 -1000 Subject: [PATCH 01/17] wip --- app/lib/data_html_parser.rb | 10 +++++++++- app/models/series.rb | 8 ++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index a7e7a2fbc..a5bd9c619 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -18,7 +18,15 @@ def get_fred_series(code, frequency = nil, aggregation_method = nil) data end - def get_bls_series(code, frequency = nil) + def get_bls_series(series_id, frequency = nil) + api_key = ENV['API_KEY_BLS'] || raise('No API key defined for BLS') + @url = 'https://api.bls.gov/publicAPI/v2/timeseries/data/%s?registration_key=%s' % [series_id, api_key] + Rails.logger.debug { "Getting data from BLS API: #{@url}" } + @doc = self.download + raise 'BLS API: empty response returned' if self.content.blank? + end + + def get_bls_series_old_DELETEME(code, frequency = nil) @code = code @url = 'https://data.bls.gov/pdq/SurveyOutputServlet' @post_parameters = { diff --git a/app/models/series.rb b/app/models/series.rb index e65c23949..a6df1365b 100644 --- a/app/models/series.rb +++ b/app/models/series.rb @@ -770,11 +770,11 @@ def Series.load_api_bea(frequency, dataset, parameters) Series.new_transformation(name, series_data, frequency) end - def Series.load_api_bls(code, frequency) - series_data = DataHtmlParser.new.get_bls_series(code, frequency) - name = "loaded series code: #{code} from BLS API" + def Series.load_api_bls(series_id, frequency) + series_data = DataHtmlParser.new.get_bls_series(series_id, frequency) + name = "loaded series code: #{series_id} from BLS API" if series_data && series_data.empty? - name = "No data collected from BLS API for #{code} freq=#{frequency} - possibly redacted" + name = "No data collected from BLS API for #{series_id} freq=#{frequency} - possibly redacted" end new_transformation(name, series_data, frequency) end From 595eaada62a314aa820f8b8c4b6e6ff1f599b822 Mon Sep 17 00:00:00 2001 From: dji Date: Wed, 1 Mar 2023 01:10:29 -1000 Subject: [PATCH 02/17] wip --- app/models/series.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/models/series.rb b/app/models/series.rb index a6df1365b..e6afff233 100644 --- a/app/models/series.rb +++ b/app/models/series.rb @@ -771,12 +771,14 @@ def Series.load_api_bea(frequency, dataset, parameters) end def Series.load_api_bls(series_id, frequency) + dhp = DataHtmlParser.new series_data = DataHtmlParser.new.get_bls_series(series_id, frequency) - name = "loaded series code: #{series_id} from BLS API" - if series_data && series_data.empty? - name = "No data collected from BLS API for #{series_id} freq=#{frequency} - possibly redacted" + link = 'API URL' % dhp.url + name = "loaded data set from #{link} with parameters shown" + if series_data.empty? + name = "No data collected from #{link} - possibly redacted" end - new_transformation(name, series_data, frequency) + Series.new_transformation(name, series_data, frequency) end def Series.load_api_fred(code, frequency = nil, aggregation_method = nil) From 0b6e30ab448782db71e5369c4d71225361443786 Mon Sep 17 00:00:00 2001 From: dji Date: Fri, 7 Apr 2023 22:35:52 -1000 Subject: [PATCH 03/17] wip --- app/lib/data_html_parser.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index 279d3c14e..d73b7f0c0 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -20,7 +20,9 @@ def get_fred_series(code, frequency = nil, aggregation_method = nil) def get_bls_series(series_id, frequency = nil) api_key = ENV['API_KEY_BLS'] || raise('No API key defined for BLS') - @url = 'https://api.bls.gov/publicAPI/v2/timeseries/data/%s?registration_key=%s' % [series_id, api_key] + thisyear = Date.today.year + @url = 'https://api.bls.gov/publicAPI/v2/timeseries/data/%s?registration_key=%s&startyear=%d&endyear=%d' % + [series_id, api_key, thisyear - 4, thisyear] Rails.logger.debug { "Getting data from BLS API: #{@url}" } @doc = self.download raise 'BLS API: empty response returned' if self.content.blank? From 48c131f7f4511609676dd30eb154153343ee0225 Mon Sep 17 00:00:00 2001 From: dji Date: Sat, 8 Apr 2023 17:12:02 -1000 Subject: [PATCH 04/17] wi --- app/lib/data_html_parser.rb | 3 ++- app/models/series.rb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index d73b7f0c0..19a132f25 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -22,10 +22,11 @@ def get_bls_series(series_id, frequency = nil) api_key = ENV['API_KEY_BLS'] || raise('No API key defined for BLS') thisyear = Date.today.year @url = 'https://api.bls.gov/publicAPI/v2/timeseries/data/%s?registration_key=%s&startyear=%d&endyear=%d' % - [series_id, api_key, thisyear - 4, thisyear] + [series_id, api_key, thisyear - 5, thisyear] Rails.logger.debug { "Getting data from BLS API: #{@url}" } @doc = self.download raise 'BLS API: empty response returned' if self.content.blank? + ### finish writing... end def get_bls_series_old_DELETEME(code, frequency = nil) diff --git a/app/models/series.rb b/app/models/series.rb index 6c490da49..ba567648e 100644 --- a/app/models/series.rb +++ b/app/models/series.rb @@ -772,7 +772,7 @@ def Series.load_api_bea(frequency, dataset, parameters) def Series.load_api_bls(series_id, frequency) dhp = DataHtmlParser.new - series_data = DataHtmlParser.new.get_bls_series(series_id, frequency) + series_data = dhp.get_bls_series(series_id, frequency) link = 'API URL' % dhp.url name = "loaded data set from #{link} with parameters shown" if series_data.empty? From 66ed5a332e6e1ca6a64d9456d4ae68bf2b0af177 Mon Sep 17 00:00:00 2001 From: dji Date: Thu, 13 Apr 2023 18:03:25 -1000 Subject: [PATCH 05/17] wip --- app/lib/data_html_parser.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index 19a132f25..008493a82 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -22,11 +22,24 @@ def get_bls_series(series_id, frequency = nil) api_key = ENV['API_KEY_BLS'] || raise('No API key defined for BLS') thisyear = Date.today.year @url = 'https://api.bls.gov/publicAPI/v2/timeseries/data/%s?registration_key=%s&startyear=%d&endyear=%d' % - [series_id, api_key, thisyear - 5, thisyear] + [series_id, api_key, thisyear - 9, thisyear] Rails.logger.debug { "Getting data from BLS API: #{@url}" } @doc = self.download raise 'BLS API: empty response returned' if self.content.blank? - ### finish writing... + json = JSON.parse(self.content) rescue raise('BLS API: JSON parse failure') + if json['status'] !~ /succeeded/i + raise 'BLS API error: %s' % json['message'].join(' ') + end + results_data = json['Results']['series'][0]['data'] ## :eyeroll + if results_data.empty? + raise 'BLS API error: %s' % json['message'].join(' ') + end + + new_data = {} + results_data.each do |dp| + ## + end + new_data end def get_bls_series_old_DELETEME(code, frequency = nil) From edf9aed173623a5a903a7bcb12516f40bec2a56b Mon Sep 17 00:00:00 2001 From: dji Date: Thu, 13 Apr 2023 18:14:52 -1000 Subject: [PATCH 06/17] wip --- app/lib/data_html_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index 008493a82..447e0d64c 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -37,7 +37,7 @@ def get_bls_series(series_id, frequency = nil) new_data = {} results_data.each do |dp| - ## + new_data[ get_date(dp['year'], dp['period']) ] = dp['value'].gsub(',','').to_f end new_data end From a16de42bcff51aa8b47703e307093763761801b0 Mon Sep 17 00:00:00 2001 From: dji Date: Thu, 13 Apr 2023 18:21:13 -1000 Subject: [PATCH 07/17] wip --- app/lib/data_html_parser.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index 447e0d64c..bd665ce23 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -31,9 +31,7 @@ def get_bls_series(series_id, frequency = nil) raise 'BLS API error: %s' % json['message'].join(' ') end results_data = json['Results']['series'][0]['data'] ## :eyeroll - if results_data.empty? - raise 'BLS API error: %s' % json['message'].join(' ') - end + raise 'BLS API error: %s' % json['message'].join(' ') if results_data.empty? new_data = {} results_data.each do |dp| From 7e44def4ac5a88996800c54d49c1edeb28949490 Mon Sep 17 00:00:00 2001 From: dji Date: Thu, 13 Apr 2023 18:24:47 -1000 Subject: [PATCH 08/17] wip --- app/lib/data_html_parser.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index bd665ce23..0af3a2806 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -27,7 +27,7 @@ def get_bls_series(series_id, frequency = nil) @doc = self.download raise 'BLS API: empty response returned' if self.content.blank? json = JSON.parse(self.content) rescue raise('BLS API: JSON parse failure') - if json['status'] !~ /succeeded/i + unless json['status'] =~ /succeeded/i raise 'BLS API error: %s' % json['message'].join(' ') end results_data = json['Results']['series'][0]['data'] ## :eyeroll @@ -35,7 +35,8 @@ def get_bls_series(series_id, frequency = nil) new_data = {} results_data.each do |dp| - new_data[ get_date(dp['year'], dp['period']) ] = dp['value'].gsub(',','').to_f + obs_date = get_date(dp['year'], dp['period']) + new_data[obs_date] = dp['value'].gsub(',','').to_f end new_data end From 6b75ecc562f61563bc834c58eac78282a39e9a69 Mon Sep 17 00:00:00 2001 From: dji Date: Thu, 13 Apr 2023 18:30:33 -1000 Subject: [PATCH 09/17] wip --- app/lib/data_html_parser.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index 0af3a2806..f314e175a 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -41,7 +41,9 @@ def get_bls_series(series_id, frequency = nil) new_data end - def get_bls_series_old_DELETEME(code, frequency = nil) + ## Should be obsolete now, but let's keep it around a while longer just in case a problem arises + ## with the new routine above + def get_bls_series_DELETEME(code, frequency = nil) @code = code @url = 'https://data.bls.gov/pdq/SurveyOutputServlet' @post_parameters = { From f87fa28fab89d57857fe37d997bd2f0f14c724e2 Mon Sep 17 00:00:00 2001 From: dji Date: Sat, 17 Jun 2023 20:20:16 -1000 Subject: [PATCH 10/17] wip --- app/lib/data_html_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index 634be795e..cd9fe4e76 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -19,7 +19,7 @@ def get_fred_series(code, frequency = nil, aggregation_method = nil) data end - def get_bls_series(series_id, frequency = nil) + def get_bls_series(series_id, _ = nil) api_key = ENV['API_KEY_BLS'] || raise('No API key defined for BLS') thisyear = Date.today.year @url = 'https://api.bls.gov/publicAPI/v2/timeseries/data/%s?registration_key=%s&startyear=%d&endyear=%d' % From d439d0755606583e227e18c6ce05a30b1c72cf76 Mon Sep 17 00:00:00 2001 From: dji Date: Sat, 17 Jun 2023 20:24:20 -1000 Subject: [PATCH 11/17] wip --- app/models/series.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/series.rb b/app/models/series.rb index 0da2ac9cf..26c9c599a 100644 --- a/app/models/series.rb +++ b/app/models/series.rb @@ -798,7 +798,7 @@ def Series.load_api_bls(series_id, frequency) series_data = dhp.get_bls_series(series_id, frequency) link = 'API URL' % dhp.url name = "loaded data set from #{link} with parameters shown" - if series_data.empty? + if series_data && series_data.empty? name = "No data collected from #{link} - possibly redacted" end Series.new_transformation(name, series_data, frequency) From 2c81592d107766b2bea4a3cad0f5022c43124f97 Mon Sep 17 00:00:00 2001 From: dji Date: Sat, 17 Jun 2023 20:35:15 -1000 Subject: [PATCH 12/17] wip --- app/lib/data_html_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index cd9fe4e76..b4669b2b3 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -36,7 +36,7 @@ def get_bls_series(series_id, _ = nil) new_data = {} results_data.each do |dp| - obs_date = get_date(dp['year'], dp['period']) + obs_date = grok_date(dp['year'], dp['period']) new_data[obs_date] = dp['value'].gsub(',','').to_f end new_data From e0f807d876bdbfa122e4c76d4a7bd3943806ebaf Mon Sep 17 00:00:00 2001 From: dji Date: Sat, 17 Jun 2023 20:42:55 -1000 Subject: [PATCH 13/17] wip --- app/lib/data_html_parser.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/lib/data_html_parser.rb b/app/lib/data_html_parser.rb index b4669b2b3..1020fa365 100644 --- a/app/lib/data_html_parser.rb +++ b/app/lib/data_html_parser.rb @@ -36,8 +36,7 @@ def get_bls_series(series_id, _ = nil) new_data = {} results_data.each do |dp| - obs_date = grok_date(dp['year'], dp['period']) - new_data[obs_date] = dp['value'].gsub(',','').to_f + new_data[ grok_date(dp['year'], dp['period']) ] = dp['value'].gsub(',','').to_f end new_data end From 113a6386e1a8e745e722789d07df8054476dfc37 Mon Sep 17 00:00:00 2001 From: dji Date: Sat, 17 Jun 2023 20:44:33 -1000 Subject: [PATCH 14/17] wip --- app/models/series.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/series.rb b/app/models/series.rb index 26c9c599a..0da2ac9cf 100644 --- a/app/models/series.rb +++ b/app/models/series.rb @@ -798,7 +798,7 @@ def Series.load_api_bls(series_id, frequency) series_data = dhp.get_bls_series(series_id, frequency) link = 'API URL' % dhp.url name = "loaded data set from #{link} with parameters shown" - if series_data && series_data.empty? + if series_data.empty? name = "No data collected from #{link} - possibly redacted" end Series.new_transformation(name, series_data, frequency) From 897f26699b75a9a39ffe84d4b4bea4cecad6a393 Mon Sep 17 00:00:00 2001 From: dji Date: Mon, 24 Jul 2023 18:08:34 -1000 Subject: [PATCH 15/17] wip --- app/models/data_source.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/data_source.rb b/app/models/data_source.rb index 1c4599e4f..1aee4bb98 100644 --- a/app/models/data_source.rb +++ b/app/models/data_source.rb @@ -236,7 +236,7 @@ def reload_source(clear_first = clear_before_load?) self.reload if presave_hook ## it sucks to have to do this, but presave_hook might change something, that will end up saved below self.update!(update_props) end - Rails.logger.info { "Completed reload of definition #{id} for series <#{self.series}> [#{description}]" } + Rails.logger.info { "Completed reload of definition #{id} for series #{series.id} <#{series}> [#{description}]" } true end From 8c3ce6888ab2a3791c3851461f91fec8bd830220 Mon Sep 17 00:00:00 2001 From: dji Date: Sun, 24 Sep 2023 15:09:55 -0600 Subject: [PATCH 16/17] a bug fix that can wait a little --- app/models/series.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/series.rb b/app/models/series.rb index c41d79d3b..08c15b701 100644 --- a/app/models/series.rb +++ b/app/models/series.rb @@ -1054,8 +1054,8 @@ def Series.search(input_string, limit: 10000, user: nil) when /^[@]/ all = all.joins(:geography) geos = tane.split(','). - map {|g| g.upcase == 'HIALL' ? %w{HI5 NBI MOL MAUI LAN HAWH HAWK} : g }. - map {|g| g.upcase == 'HI5' ? %w{HI CNTY} : g }. + map {|g| g.upcase == 'HIALL' ? %w{HI5 NBI MOL MAUI LAN HAWH HAWK} : g }.flatten. + map {|g| g.upcase == 'HI5' ? %w{HI CNTY} : g }.flatten. map {|g| g.upcase == 'CNTY' ? %w{HAW HON KAU MAU} : g }.flatten Rails.logger.info "-------------------> geos = #{geos.join(',')}" qmarks = (['?'] * geos.count).join(',') From c281c3f2195b84fed4bf170dbf74b43c91443707 Mon Sep 17 00:00:00 2001 From: dji Date: Tue, 26 Sep 2023 21:04:33 -0600 Subject: [PATCH 17/17] wip --- app/models/series.rb | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/app/models/series.rb b/app/models/series.rb index 08c15b701..0960c8b90 100644 --- a/app/models/series.rb +++ b/app/models/series.rb @@ -884,8 +884,13 @@ def Series.load_api_dvw(mod, freq, indicator, dimensions) end def daily_census - raise 'Cannot compute avg daily census on daily series' if frequency == 'day' - self / (is_SA? ? 30.42 : days_in_period) ## 30.42 is 365/12, the average number of days/month + if is_SA? + fpf = freq_per_freq(frequency, :year) || raise("Cannot compute ADC on SA series of frequency #{frequency}") + denom = (365 / fpf.to_f).round(2) + else + denom = days_in_period + end + self / denom end def days_in_period