Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion app/controllers/statcan_datasets_controller.rb
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
class StatcanDatasetsController < ApplicationController
def show
dataset = StatcanDataset.find_by(name: params[:id])
render json: dataset
if dataset
render json: dataset
else
render json: { error: "Not found" }, status: :not_found
end
end
end
4 changes: 4 additions & 0 deletions app/models/statcan_dataset.rb
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@ class StatcanDataset < ApplicationRecord
validates :sync_schedule, presence: true
validate :valid_cron_expression

def to_param
name
end

def self.filter_stale(datasets, current_time = Time.current)
datasets.select { |dataset| dataset.needs_sync?(current_time) }
end
Expand Down
2 changes: 1 addition & 1 deletion app/views/avo/scraping_health/index.html.erb
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@
<%= entry.published_at ? time_ago_in_words(entry.published_at) + " ago" : "—" %>
</td>
<td class="px-4 py-3 text-sm text-gray-500 max-w-xs truncate">
<%= link_to entry.url, entry.url, target: "_blank", class: "text-blue-600 hover:underline", title: entry.url %>
<%= link_to entry.url, entry.url.to_s.match?(/\Ahttps?:\/\//) ? entry.url : "#", target: "_blank", class: "text-blue-600 hover:underline", title: entry.url %>
</td>
</tr>
<% end %>
Expand Down
8 changes: 4 additions & 4 deletions test/jobs/commitment_relevance_filter_job_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,13 @@ class CommitmentRelevanceFilterJobTest < ActiveJob::TestCase
assert_not_includes result.to_a, commitment
end

test "excludes abandoned commitments regardless of date" do
test "excludes broken commitments regardless of date" do
commitment = Commitment.create!(
government: @government,
title: "Abandoned commitment",
description: "This was abandoned",
title: "Broken commitment",
description: "This was broken",
commitment_type: :spending,
status: :abandoned,
status: :broken,
date_promised: Date.new(2025, 1, 1)
)

Expand Down
6 changes: 3 additions & 3 deletions test/jobs/statcan_cron_job_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ def setup
end

test "should enqueue sync jobs for stale datasets only" do
current_time = Time.parse("2025-01-02 14:00:00") # 2pm
current_time = Time.utc(2025, 1, 2, 14, 0, 0) # 2pm UTC

# Create a stale dataset (never synced)
stale_dataset1 = StatcanDataset.create!(
Expand All @@ -22,15 +22,15 @@ def setup
name: "stale-old-sync",
statcan_url: "https://statcan.gc.ca/stale2.csv",
sync_schedule: "0 0 * * *",
last_synced_at: Time.parse("2025-01-01 23:00:00") # Yesterday 11pm
last_synced_at: Time.utc(2025, 1, 1, 23, 0, 0) # Yesterday 11pm UTC
)

# Create a fresh dataset (recent sync)
_fresh_dataset = StatcanDataset.create!(
name: "fresh-dataset",
statcan_url: "https://statcan.gc.ca/fresh.csv",
sync_schedule: "0 0 * * *",
last_synced_at: Time.parse("2025-01-02 01:00:00") # 1am today
last_synced_at: Time.utc(2025, 1, 2, 1, 0, 0) # 1am UTC today
)

# Track enqueued jobs
Expand Down
4 changes: 2 additions & 2 deletions test/models/statcan_dataset_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,9 @@ def self.valid_attributes
end

test "needs_sync returns true when last sync was before last scheduled time" do
attributes = self.class.valid_attributes.merge(sync_schedule: "0 0 * * *", last_synced_at: Time.parse("2025-01-01 23:00:00"))
attributes = self.class.valid_attributes.merge(sync_schedule: "0 0 * * *", last_synced_at: Time.utc(2025, 1, 1, 23, 0, 0))
dataset = StatcanDataset.new(attributes)
current_time = Time.parse("2025-01-02 14:00:00") # 2pm next day
current_time = Time.utc(2025, 1, 2, 14, 0, 0) # 2pm next day UTC

assert dataset.needs_sync?(current_time)
end
Expand Down