diff --git a/Gemfile b/Gemfile
index 0d88c8f..d0e6388 100644
--- a/Gemfile
+++ b/Gemfile
@@ -9,7 +9,7 @@ gem "puma", ">= 5.0"
# Use GoodJob for Active Job queue adapter
gem "good_job", "~> 4.10", ">= 4.10.2"
# Build JSON APIs with ease [https://github.com/rails/jbuilder]
-# gem "jbuilder"
+gem "jbuilder"
# Use Active Model has_secure_password [https://guides.rubyonrails.org/active_model_basics.html#securepassword]
# gem "bcrypt", "~> 3.1.7"
@@ -46,7 +46,8 @@ gem "importmap-rails", "~> 2.1"
gem "propshaft", "~> 1.1"
-gem "ruby_llm", "~> 1.3"
+gem "ruby_llm", "~> 1.3", github: "xrendan/ruby_llm", branch: "structured-output"
+# gem "ruby_llm", "~> 1.3", path: "../ruby_llm"
gem "dotenv", groups: [ :development, :test ]
@@ -55,3 +56,5 @@ gem "feedjira", "~> 3.2"
gem "http", "~> 5.3"
gem "iconv", "~> 1.1"
+
+gem "structify", "~> 0.3.4"
diff --git a/Gemfile.lock b/Gemfile.lock
index 03a81be..6333224 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,18 @@
+GIT
+ remote: https://github.com/xrendan/ruby_llm.git
+ revision: c849a1ef77e533190173dc7d6aa274e0f491b29f
+ branch: structured-output
+ specs:
+ ruby_llm (1.3.1)
+ base64
+ event_stream_parser (~> 1)
+ faraday (>= 1.10.0)
+ faraday-multipart (>= 1)
+ faraday-net_http (>= 1)
+ faraday-retry (>= 1)
+ marcel (~> 1.0)
+ zeitwerk (~> 2)
+
GEM
remote: https://rubygems.org/
specs:
@@ -78,6 +93,8 @@ GEM
addressable (2.8.7)
public_suffix (>= 2.0.2, < 7.0)
ast (2.4.3)
+ attr_json (2.5.0)
+ activerecord (>= 6.0.0, < 8.1)
avo (3.21.1)
actionview (>= 6.1)
active_link_to
@@ -185,6 +202,9 @@ GEM
pp (>= 0.6.0)
rdoc (>= 4.0.0)
reline (>= 0.4.2)
+ jbuilder (2.13.0)
+ actionview (>= 5.0.0)
+ activesupport (>= 5.0.0)
json (2.12.2)
language_server-protocol (3.17.0.5)
lint_roller (1.1.0)
@@ -339,18 +359,12 @@ GEM
rubocop-performance (>= 1.24)
rubocop-rails (>= 2.30)
ruby-progressbar (1.13.0)
- ruby_llm (1.3.1)
- base64
- event_stream_parser (~> 1)
- faraday (>= 1.10.0)
- faraday-multipart (>= 1)
- faraday-net_http (>= 1)
- faraday-retry (>= 1)
- marcel (~> 1.0)
- zeitwerk (~> 2)
sax-machine (1.3.2)
securerandom (0.4.1)
stringio (3.1.7)
+ structify (0.3.4)
+ activesupport (>= 7.0, < 9.0)
+ attr_json (~> 2.1)
thor (1.3.2)
timeout (0.4.3)
turbo-rails (2.0.16)
@@ -401,12 +415,14 @@ DEPENDENCIES
http (~> 5.3)
iconv (~> 1.1)
importmap-rails (~> 2.1)
+ jbuilder
pg (~> 1.1)
propshaft (~> 1.1)
puma (>= 5.0)
rails (~> 8.0.2)
rubocop-rails-omakase
- ruby_llm (~> 1.3)
+ ruby_llm (~> 1.3)!
+ structify (~> 0.3.4)
tzinfo-data
BUNDLED WITH
diff --git a/README.md b/README.md
index 9662b05..ebd00fc 100644
--- a/README.md
+++ b/README.md
@@ -1,14 +1,45 @@
# Government Outcomes Tracker API
-Data Model
+## Data Model
`Government`:
- Think of this as the core tenant. This is currently unused, but is useful for future development where we want to track
+ Think of this as the core tenant. All records are currently linked to the federal government,
+ but is useful for future development where we want to track
provincial governments in addition to the federal government.
+`Department`:
+ Departments are the units within a government. They are associated with a government.
+
+`Minister`:
+ Ministers are the individuals who are responsible for a department. They are associated with a department.
+ Each department may have multiple ministers. They have a start and end date. There may also be multiple ministers
+ associated with a department at the same time if there is both a minister and a secretary of state.
+
+`Promise`:
+ Promises are the commitments that the government has made. They are associated with a government.
+ These were originally extracted using an LLM, but are currently static. We need to rebuild the ability
+ to extract them from source documents. They are associated with a department.
+
+`Feeds`:
+ Feeds are sources of information that we scrape to understand what the government is doing.
+ They are associated with a government.When we scrape them we generate `Entry` records
+ with the raw data. Currently we support RSS feeds, but we would like to also support
+ newsletters (using active mailbox), and scraping unstructured webpages.
+
+`Entry`:
+ Entries contain the raw data scraped from feeds. They are associated with a feed and a government.
+ Some Entries (like those scraped from the Canada Gazette RSS feeds) are indexes of other Entries,
+ those are flagged as index_entries and are not used for matching, but are then used to find other Entries
+ which are linked using the parent_entry_id column.
+
+`Activity`:
+ Activities are the actions that the government is taking. They are associated with an Entry.
+ These are extracted using an LLM from the Entry's raw data. Each entry might have multiple activities.
+
+`Evidence`:
+ Evidence links an Activity to a Promise. They are linked using an LLM.
-## Using AI
### 🛠Setup
diff --git a/app/avo/resources/activity.rb b/app/avo/resources/activity.rb
new file mode 100644
index 0000000..81bf3dc
--- /dev/null
+++ b/app/avo/resources/activity.rb
@@ -0,0 +1,19 @@
+class Avo::Resources::Activity < Avo::BaseResource
+ # self.includes = []
+ # self.attachments = []
+ # self.search = {
+ # query: -> { query.ransack(id_eq: params[:q], m: "or").result(distinct: false) }
+ # }
+
+ def fields
+ field :id, as: :id
+ field :entry, as: :belongs_to
+ field :government, as: :belongs_to
+ field :title, as: :text
+ field :details, as: :text
+ field :source_url, as: :text
+ field :info, as: :code
+ field :publication_date, as: :date
+ field :in_force_date, as: :date
+ end
+end
diff --git a/app/avo/resources/evidence.rb b/app/avo/resources/evidence.rb
index ab942c7..3447608 100644
--- a/app/avo/resources/evidence.rb
+++ b/app/avo/resources/evidence.rb
@@ -7,26 +7,17 @@ class Avo::Resources::Evidence < Avo::BaseResource
def fields
field :id, as: :id
- field :raw_gazette_notice_id, as: :text
- field :rias_summary, as: :textarea
- field :description_or_details, as: :textarea
- field :evidence_date, as: :date_time
- field :evidence_id, as: :text
- field :evidence_source_type, as: :text
- field :hybrid_linking_avg_confidence, as: :number
- field :hybrid_linking_method, as: :text
- field :hybrid_linking_timestamp, as: :date_time
- field :ingested_at, as: :date_time
- field :parliament_session_id, as: :text
- field :promise_linking_processed_at, as: :date_time
- field :promise_linking_status, as: :text
- field :promise_links_found2, as: :number
- field :source_document_raw_id, as: :text
- field :source_url, as: :text
- field :title_or_summary, as: :textarea
- field :key_concepts, as: :textarea
- field :linked_departments, as: :textarea
- field :promise_ids, as: :textarea
- field :llm_analysis_raw, as: :code
+ field :activity, as: :belongs_to
+ field :promise, as: :belongs_to
+ field :impact, as: :text
+ field :impact_magnitude, as: :number
+ field :impact_reason, as: :text
+ field :linked_at, as: :date_time
+ field :linked_by, as: :belongs_to
+ field :link_type, as: :text
+ field :link_reason, as: :text
+ field :review, as: :boolean
+ field :reviewed_by, as: :belongs_to
+ field :reviewed_at, as: :date_time
end
end
diff --git a/app/controllers/activities_controller.rb b/app/controllers/activities_controller.rb
new file mode 100644
index 0000000..65b72cc
--- /dev/null
+++ b/app/controllers/activities_controller.rb
@@ -0,0 +1,21 @@
+class ActivitiesController < ApplicationController
+ before_action :set_activity, only: %i[ show ]
+
+ # GET /activities
+ def index
+ @activities = Activity.all
+
+ render json: @activities
+ end
+
+ # GET /activities/1
+ def show
+ render json: @activity
+ end
+
+ private
+ # Use callbacks to share common setup or constraints between actions.
+ def set_activity
+ @activity = Activity.find(params.expect(:id))
+ end
+end
diff --git a/app/controllers/avo/activities_controller.rb b/app/controllers/avo/activities_controller.rb
new file mode 100644
index 0000000..58a1eb1
--- /dev/null
+++ b/app/controllers/avo/activities_controller.rb
@@ -0,0 +1,4 @@
+# This controller has been generated to enable Rails' resource routes.
+# More information on https://docs.avohq.io/3.0/controllers.html
+class Avo::ActivitiesController < Avo::ResourcesController
+end
diff --git a/app/controllers/evidences_controller.rb b/app/controllers/evidences_controller.rb
index ac3d439..003302c 100644
--- a/app/controllers/evidences_controller.rb
+++ b/app/controllers/evidences_controller.rb
@@ -1,11 +1,21 @@
class EvidencesController < ApplicationController
+ before_action :set_evidence, only: %i[ show ]
+
+ # GET /evidences
def index
@evidences = Evidence.all
+
render json: @evidences
end
+ # GET /evidences/1
def show
- @evidence = Evidence.find(params[:id])
render json: @evidence
end
+
+ private
+ # Use callbacks to share common setup or constraints between actions.
+ def set_evidence
+ @evidence = Evidence.find(params.expect(:id))
+ end
end
diff --git a/app/jobs/feed_refresher_job.rb b/app/jobs/feed_refresher_job.rb
new file mode 100644
index 0000000..9dac3bd
--- /dev/null
+++ b/app/jobs/feed_refresher_job.rb
@@ -0,0 +1,11 @@
+class FeedRefresherJob < ApplicationJob
+ queue_as :default
+
+ def perform(feed = nil)
+ if feed.nil?
+ Feed.all.each { |f| FeedRefresherJob.perform_later(f) }
+ else
+ feed.refresh!
+ end
+ end
+end
diff --git a/app/models/activity.rb b/app/models/activity.rb
new file mode 100644
index 0000000..85a714d
--- /dev/null
+++ b/app/models/activity.rb
@@ -0,0 +1,7 @@
+class Activity < ApplicationRecord
+ belongs_to :entry
+ belongs_to :government
+
+ has_many :evidences
+ has_many :promises, through: :evidences
+end
diff --git a/app/models/activity_extractor.rb b/app/models/activity_extractor.rb
new file mode 100644
index 0000000..65ef0fc
--- /dev/null
+++ b/app/models/activity_extractor.rb
@@ -0,0 +1,97 @@
+class ActivityExtractor < Chat
+ include Structify::Model
+
+ def prompt(promises, entry)
+ <<~PROMPT
+ You are tasked with extracting activities from political artifacts that could impact the progress of promises made by the government. This analysis will help track the government's actions and their potential effects on fulfilling their commitments.
+
+ First, review the list of government promises:
+
+ #{promises.map(&:format_for_llm).join("\n")}
+
+
+ Now, carefully read and analyze the following political artifact:
+ #{entry.format_for_llm}
+
+ Your task is to extract activities mentioned in the political artifact that could potentially impact the progress of the government promises listed above. Follow these steps:
+
+ 1. Identify any actions, initiatives, policies, or decisions mentioned in the artifact.
+ 2. For each identified activity, determine if it relates to any of the government promises.
+ 3. Assess whether the activity could have a positive, negative, or neutral impact on the progress of the related promise(s).
+
+ When analyzing the impact:
+ - Consider direct and indirect effects
+ - Think about short-term and long-term consequences
+ - Take into account the scale and scope of the activity
+
+ Present your findings in the following format:
+
+
+
+ [Describe the activity]
+ [List the related government promise(s)]
+ [Explain the potential impact on the promise(s), whether positive, negative, or neutral, and why]
+
+
+
+ If no relevant activities are found in the political artifact, state this clearly in your response.
+
+ Try to minimize the number of activities listed, they should be combined if they are similar enough to avoid redundancy.
+
+ Remember to focus only on activities that could potentially impact the government's progress on their promises. Do not include unrelated information or speculate beyond what is reasonably implied by the artifact.
+ PROMPT
+ end
+
+ schema_definition do
+ version 1
+ name "ActivityExtraction"
+ description "Extracts activities from an entry"
+ field :reason_for_no_activities, :string, description: "Reason for the activity extraction"
+ field :activities, :array,
+ description: "List of activities that the government has impacted one of the promises",
+ items: {
+ type: "object", properties: {
+ "title" => { type: "string" },
+ "summary" => { type: "string", description: "Summary of what the government has done or proposed to do" },
+ "impacted_promises" => { type: "array", items: { type: "object", properties: {
+ "promise_id" => { type: "string", description: "The ID of the promise that the activity impacts" },
+ "potential_impact" => { type: "string", enum: [ "positive", "negative", "neutral" ] },
+ "potential_impact_magnitude" => { type: "integer", description: "The magnitude of the potential impact on the promise(s). 1 indicates a minor impact, 2 indicates a moderate impact, and 3 indicates a significant impact." },
+ "potential_impact_reason" => { type: "string", description: "Explain the potential impact on the promise(s), whether positive, negative, or neutral, and why" }
+ } } }
+ }
+ }
+ end
+
+ def extract_activities!
+ raise ArgumentError.new("Record is not provided") unless self.record and self.record.is_a?(Entry)
+ p = prompt(
+ Promise.all,
+ self.record
+ )
+
+ self.extract! p
+
+
+ activities.each do |activity|
+ rec = Activity.create!(
+ government_id: self.record.government_id,
+ entry: self.record,
+ title: activity["title"],
+ summary: activity["summary"],
+ published_at: self.record.published_at
+ )
+
+ activity["impacted_promises"].each do |impacted_promise|
+ rec.evidences.create!(
+ promise_id: Promise.find_by!(promise_id: impacted_promise["promise_id"]).id,
+ linked_at: Time.now,
+ link_type: "automated",
+ impact: impacted_promise["potential_impact"],
+ impact_magnitude: impacted_promise["potential_impact_magnitude"],
+ impact_reason: impacted_promise["potential_impact_reason"]
+ )
+ end
+ end
+ end
+end
diff --git a/app/models/chat.rb b/app/models/chat.rb
index 03d2906..08290ec 100644
--- a/app/models/chat.rb
+++ b/app/models/chat.rb
@@ -1,5 +1,24 @@
class Chat < ApplicationRecord
acts_as_chat
- belongs_to :record, polymorphic: true
+ belongs_to :record, polymorphic: true, optional: true
+
+ def system_prompt
+ end
+
+ def extract!(prompt)
+ raise unless self.class.respond_to?(:json_schema)
+
+ chat = self.with_schema(self.class.json_schema)
+
+ if self.system_prompt.present?
+ chat = chat.with_instructions(self.system_prompt)
+ end
+
+ message = chat.ask(prompt)
+
+ attributes = JSON.parse(message.content)
+
+ self.update(attributes)
+ end
end
diff --git a/app/models/entry.rb b/app/models/entry.rb
index 04e5d1f..84dbe15 100644
--- a/app/models/entry.rb
+++ b/app/models/entry.rb
@@ -1,6 +1,9 @@
class Entry < ApplicationRecord
+ include Structify::Model
+
belongs_to :feed
belongs_to :government
+ has_many :activity_extractors, as: :record
after_commit :fetch_data!, on: [ :create ]
@@ -45,4 +48,22 @@ def fetch_data!(in_background: true)
def create_subentries!
# Some data sources (like the canada gazette have an RSS feed that is just an index of all the entries, so we need to fetch the actual entries from the feed)
end
+
+ def extract_activities!
+ extractor = ActivityExtractor.create!(record: self)
+ extractor.extract_activities!
+ self.activities_extracted_at = Time.now
+ end
+
+ def format_for_llm
+ <<~XML
+
+ #{feed.title}
+ #{published_at}
+ #{title}
+ #{summary}
+ #{parsed_markdown}
+
+ XML
+ end
end
diff --git a/app/models/evidence.rb b/app/models/evidence.rb
index 0ea04a1..f020ce8 100644
--- a/app/models/evidence.rb
+++ b/app/models/evidence.rb
@@ -1,2 +1,6 @@
class Evidence < ApplicationRecord
+ belongs_to :activity
+ belongs_to :promise
+ belongs_to :linked_by, class_name: "User", optional: true
+ belongs_to :reviewed_by, class_name: "User", optional: true
end
diff --git a/app/models/feed.rb b/app/models/feed.rb
index b3dc531..93ba723 100644
--- a/app/models/feed.rb
+++ b/app/models/feed.rb
@@ -20,8 +20,8 @@ def refresh!
def create_entries!(entries)
entries.each do |entry|
+ raise "URL is required" unless entry.url.present?
Entry.find_or_create_by!(government: government, feed: self, url: entry.url) do |rec|
- puts
rec.title = entry.title
rec.summary = entry.summary
rec.published_at = entry.published
diff --git a/app/models/promise.rb b/app/models/promise.rb
index 21aeb35..08d18f0 100644
--- a/app/models/promise.rb
+++ b/app/models/promise.rb
@@ -1,2 +1,10 @@
class Promise < ApplicationRecord
+ has_many :evidences, dependent: :destroy
+ def format_for_llm
+ {
+ promise_id: promise_id,
+ title: concise_title,
+ description: description
+ }
+ end
end
diff --git a/config/application.rb b/config/application.rb
index 5e10d07..b432fc9 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -41,11 +41,13 @@ class Application < Rails::Application
# Skip views, helpers and assets when generating a new resource.
config.api_only = false
-
- # Re-enable middleware needed for the Good Job Dashboard
- # config.middleware.use Rack::MethodOverride
- # config.middleware.use ActionDispatch::Flash
- # config.middleware.use ActionDispatch::Cookies
- # config.middleware.use ActionDispatch::Session::CookieStore
+ config.good_job.cron = {
+ feed_refresh: { # each recurring job must have a unique key
+ cron: "every 3 hours", # cron-style scheduling format by fugit gem
+ class: "FeedRefreshJob", # name of the job class as a String; must reference an Active Job job class
+ description: "Refreshed feeds and creates new entries", # optional description that appears in Dashboard,
+ enabled_by_default: -> { Rails.env.production? } # Only enable in production, otherwise can be enabled manually through Dashboard
+ }
+ }
end
end
diff --git a/config/environments/production.rb b/config/environments/production.rb
index 47d2b06..18180c5 100644
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -49,6 +49,8 @@
# Replace the default in-process and non-durable queuing backend for Active Job.
config.active_job.queue_adapter = :good_job
+ config.good_job.cron_graceful_restart_period = 1.minute
+
# Ignore bad email addresses and do not raise email delivery errors.
# Set this to true and configure the email server for immediate delivery to raise delivery errors.
# config.action_mailer.raise_delivery_errors = false
diff --git a/config/routes.rb b/config/routes.rb
index d067214..e12ed4c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,4 +1,5 @@
Rails.application.routes.draw do
+ resources :evidences
if Rails.env.development?
mount GoodJob::Engine => "/admin/good_job"
mount Avo::Engine => "/admin"
@@ -10,7 +11,7 @@
end
devise_for :users
-
+ resources :activities, only: [ :index, :show ]
resources :feeds, only: [ :index, :show ]
resources :entries, only: [ :index, :show ]
resources :ministers, only: [ :index, :show ]
diff --git a/db/migrate/20250613154200_create_evidences.rb b/db/migrate/20250613154200_create_evidences.rb
deleted file mode 100644
index f532e23..0000000
--- a/db/migrate/20250613154200_create_evidences.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-class CreateEvidences < ActiveRecord::Migration[8.0]
- def change
- create_table :evidences do |t|
- t.string :raw_gazette_notice_id
- t.text :rias_summary
- t.text :description_or_details
- t.datetime :evidence_date
- t.string :evidence_id
- t.string :evidence_source_type
- t.float :hybrid_linking_avg_confidence
- t.string :hybrid_linking_method
- t.datetime :hybrid_linking_timestamp
- t.datetime :ingested_at
- t.string :parliament_session_id
- t.datetime :promise_linking_processed_at
- t.string :promise_linking_status
- t.integer :promise_links_found2
- t.string :source_document_raw_id
- t.string :source_url
- t.text :title_or_summary
- t.text :key_concepts, array: true, default: []
- t.text :linked_departments, array: true, default: []
- t.text :promise_ids, array: true, default: []
- t.jsonb :llm_analysis_raw, default: {}
-
- t.timestamps
- end
- end
-end
diff --git a/db/migrate/20250618152229_create_activities.rb b/db/migrate/20250618152229_create_activities.rb
new file mode 100644
index 0000000..889c963
--- /dev/null
+++ b/db/migrate/20250618152229_create_activities.rb
@@ -0,0 +1,15 @@
+class CreateActivities < ActiveRecord::Migration[8.0]
+ def change
+ create_table :activities do |t|
+ t.references :entry, null: false, foreign_key: true
+ t.references :government, null: false, foreign_key: true
+ t.string :title
+ t.string :summary
+ t.string :source_url
+ t.jsonb :info
+ t.timestamp :published_at
+
+ t.timestamps
+ end
+ end
+end
diff --git a/db/migrate/20250618171826_add_json_attributes_to_chat.rb b/db/migrate/20250618171826_add_json_attributes_to_chat.rb
new file mode 100644
index 0000000..64c9714
--- /dev/null
+++ b/db/migrate/20250618171826_add_json_attributes_to_chat.rb
@@ -0,0 +1,5 @@
+class AddJsonAttributesToChat < ActiveRecord::Migration[8.0]
+ def change
+ add_column :chats, :json_attributes, :jsonb
+ end
+end
diff --git a/db/migrate/20250618223347_create_evidences.rb b/db/migrate/20250618223347_create_evidences.rb
new file mode 100644
index 0000000..fe4d32e
--- /dev/null
+++ b/db/migrate/20250618223347_create_evidences.rb
@@ -0,0 +1,22 @@
+class CreateEvidences < ActiveRecord::Migration[8.0]
+ def change
+ drop_table :evidences, if_exists: true
+
+ create_table :evidences do |t|
+ t.references :activity, null: false, foreign_key: true
+ t.references :promise, null: false, foreign_key: true
+ t.string :impact
+ t.integer :impact_magnitude
+ t.string :impact_reason
+ t.timestamp :linked_at
+ t.references :linked_by, null: true, foreign_key: { to_table: :users }
+ t.string :link_type
+ t.string :link_reason
+ t.boolean :review
+ t.references :reviewed_by, null: true, foreign_key: { to_table: :users }
+ t.timestamp :reviewed_at
+
+ t.timestamps
+ end
+ end
+end
diff --git a/db/migrate/20250618224002_add_type_to_chat.rb b/db/migrate/20250618224002_add_type_to_chat.rb
new file mode 100644
index 0000000..c331d60
--- /dev/null
+++ b/db/migrate/20250618224002_add_type_to_chat.rb
@@ -0,0 +1,5 @@
+class AddTypeToChat < ActiveRecord::Migration[8.0]
+ def change
+ add_column :chats, :type, :string
+ end
+end
diff --git a/db/migrate/20250619005535_add_activities_extracted_at_to_entries.rb b/db/migrate/20250619005535_add_activities_extracted_at_to_entries.rb
new file mode 100644
index 0000000..9042c07
--- /dev/null
+++ b/db/migrate/20250619005535_add_activities_extracted_at_to_entries.rb
@@ -0,0 +1,5 @@
+class AddActivitiesExtractedAtToEntries < ActiveRecord::Migration[8.0]
+ def change
+ add_column :entries, :activities_extracted_at, :timestamp
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6fe225f..b6b62cf 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema[8.0].define(version: 2025_06_17_020523) do
+ActiveRecord::Schema[8.0].define(version: 2025_06_19_005535) do
# These are extensions that must be enabled in order to support this database
enable_extension "pg_catalog.plpgsql"
@@ -42,12 +42,28 @@
t.index ["blob_id", "variation_digest"], name: "index_active_storage_variant_records_uniqueness", unique: true
end
+ create_table "activities", force: :cascade do |t|
+ t.bigint "entry_id", null: false
+ t.bigint "government_id", null: false
+ t.string "title"
+ t.string "summary"
+ t.string "source_url"
+ t.jsonb "info"
+ t.datetime "published_at", precision: nil
+ t.datetime "created_at", null: false
+ t.datetime "updated_at", null: false
+ t.index ["entry_id"], name: "index_activities_on_entry_id"
+ t.index ["government_id"], name: "index_activities_on_government_id"
+ end
+
create_table "chats", force: :cascade do |t|
t.string "model_id"
t.bigint "record_id"
t.string "record_type"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
+ t.jsonb "json_attributes"
+ t.string "type"
end
create_table "departments", force: :cascade do |t|
@@ -77,34 +93,30 @@
t.string "parsed_markdown"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
+ t.datetime "activities_extracted_at", precision: nil
t.index ["feed_id"], name: "index_entries_on_feed_id"
t.index ["government_id"], name: "index_entries_on_government_id"
end
create_table "evidences", force: :cascade do |t|
- t.string "raw_gazette_notice_id"
- t.text "rias_summary"
- t.text "description_or_details"
- t.datetime "evidence_date"
- t.string "evidence_id"
- t.string "evidence_source_type"
- t.float "hybrid_linking_avg_confidence"
- t.string "hybrid_linking_method"
- t.datetime "hybrid_linking_timestamp"
- t.datetime "ingested_at"
- t.string "parliament_session_id"
- t.datetime "promise_linking_processed_at"
- t.string "promise_linking_status"
- t.integer "promise_links_found2"
- t.string "source_document_raw_id"
- t.string "source_url"
- t.text "title_or_summary"
- t.text "key_concepts", default: [], array: true
- t.text "linked_departments", default: [], array: true
- t.text "promise_ids", default: [], array: true
- t.jsonb "llm_analysis_raw", default: {}
+ t.bigint "activity_id", null: false
+ t.bigint "promise_id", null: false
+ t.string "impact"
+ t.integer "impact_magnitude"
+ t.string "impact_reason"
+ t.datetime "linked_at", precision: nil
+ t.bigint "linked_by_id"
+ t.string "link_type"
+ t.string "link_reason"
+ t.boolean "review"
+ t.bigint "reviewed_by_id"
+ t.datetime "reviewed_at", precision: nil
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
+ t.index ["activity_id"], name: "index_evidences_on_activity_id"
+ t.index ["linked_by_id"], name: "index_evidences_on_linked_by_id"
+ t.index ["promise_id"], name: "index_evidences_on_promise_id"
+ t.index ["reviewed_by_id"], name: "index_evidences_on_reviewed_by_id"
end
create_table "feeds", force: :cascade do |t|
@@ -349,9 +361,15 @@
add_foreign_key "active_storage_attachments", "active_storage_blobs", column: "blob_id"
add_foreign_key "active_storage_variant_records", "active_storage_blobs", column: "blob_id"
+ add_foreign_key "activities", "entries"
+ add_foreign_key "activities", "governments"
add_foreign_key "departments", "governments"
add_foreign_key "entries", "feeds"
add_foreign_key "entries", "governments"
+ add_foreign_key "evidences", "activities"
+ add_foreign_key "evidences", "promises"
+ add_foreign_key "evidences", "users", column: "linked_by_id"
+ add_foreign_key "evidences", "users", column: "reviewed_by_id"
add_foreign_key "feeds", "governments"
add_foreign_key "messages", "chats"
add_foreign_key "messages", "tool_calls"
diff --git a/db/seeds/canada.rb b/db/seeds/canada.rb
index 975d458..adac76b 100644
--- a/db/seeds/canada.rb
+++ b/db/seeds/canada.rb
@@ -545,36 +545,4 @@
puts "Seeding Evidences..."
-evidence_data = {
- raw_gazette_notice_id: "GN-2024-001",
- rias_summary: "This notice outlines planned policy updates...",
- description_or_details: "Full text of the Gazette outlining broadband expansion...",
- evidence_date: "2024-04-20T09:00:00Z",
- evidence_id: "EVID1234",
- evidence_source_type: "gazette",
- hybrid_linking_avg_confidence: 0.87,
- hybrid_linking_method: "AI + manual",
- hybrid_linking_timestamp: "2024-04-21T10:00:00Z",
- ingested_at: "2024-04-20T12:00:00Z",
- parliament_session_id: "44-1",
- promise_linking_processed_at: "2024-04-21T12:30:00Z",
- promise_linking_status: "linked",
- promise_links_found2: 1,
- source_document_raw_id: "DOC-2024-XYZ",
- source_url: "https://example.com/evidence-doc",
- title_or_summary: "Government expands rural broadband",
- key_concepts: [ "broadband", "rural", "infrastructure" ],
- linked_departments: [ "ISED" ],
- promise_ids: [ "LPC_20250419_OTHER_71c8234b" ],
- llm_analysis_raw: {
- "summary" => "The document supports the stated promise to expand rural digital access.",
- "confidence" => 0.92
- }
-}
-
-Evidence.find_or_initialize_by(evidence_id: evidence_data[:evidence_id]).tap do |e|
- e.assign_attributes(evidence_data)
- e.save!
-end
-
puts "Done seeding"
diff --git a/db/seeds/canada_promises_2024.rb b/db/seeds/canada_promises_2024.rb
index 5b87eba..e29f8bc 100644
--- a/db/seeds/canada_promises_2024.rb
+++ b/db/seeds/canada_promises_2024.rb
@@ -89,7 +89,7 @@ def parse_firebase_timestamp(ts)
end
promises.each do |promise|
- Promise.find_or_create_by!(promise_id: promise["promise_id"]) do |e|
+ Promise.find_or_create_by!(promise_id: promise[:promise_id]) do |e|
e.assign_attributes(promise)
end
end
diff --git a/test/fixtures/evidences.yml b/test/fixtures/evidences.yml
deleted file mode 100644
index e236e7d..0000000
--- a/test/fixtures/evidences.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-# Read about fixtures at https://api.rubyonrails.org/classes/ActiveRecord/FixtureSet.html
-
-one:
- raw_gazette_notice_id: MyString
- rias_summary: MyText
- description_or_details: MyText
- evidence_date: 2025-06-13 09:42:00
- evidence_id: MyString
- evidence_source_type: MyString
- hybrid_linking_avg_confidence: 1.5
- hybrid_linking_method: MyString
- hybrid_linking_timestamp: 2025-06-13 09:42:00
- ingested_at: 2025-06-13 09:42:00
- parliament_session_id: MyString
- promise_linking_processed_at: 2025-06-13 09:42:00
- promise_linking_status: MyString
- promise_links_found2: 1
- source_document_raw_id: MyString
- source_url: MyString
- title_or_summary: MyText
-
-two:
- raw_gazette_notice_id: MyString
- rias_summary: MyText
- description_or_details: MyText
- evidence_date: 2025-06-13 09:42:00
- evidence_id: MyString
- evidence_source_type: MyString
- hybrid_linking_avg_confidence: 1.5
- hybrid_linking_method: MyString
- hybrid_linking_timestamp: 2025-06-13 09:42:00
- ingested_at: 2025-06-13 09:42:00
- parliament_session_id: MyString
- promise_linking_processed_at: 2025-06-13 09:42:00
- promise_linking_status: MyString
- promise_links_found2: 1
- source_document_raw_id: MyString
- source_url: MyString
- title_or_summary: MyText