From f7c16f5ef3150a2121801258e4225e18695efedd Mon Sep 17 00:00:00 2001 From: Adam McKerlie Date: Wed, 2 Jul 2025 13:47:09 -0400 Subject: [PATCH 1/3] Adds PromiseProgressUpdaterJob that updates the promise progress based on the attached evidence --- app/jobs/promise_progress_updater_job.rb | 7 ++ app/models/evidence.rb | 12 +++ app/models/promise.rb | 17 ++- app/models/promise_progress_updater.rb | 131 +++++++++++++++++++++++ 4 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 app/jobs/promise_progress_updater_job.rb create mode 100644 app/models/promise_progress_updater.rb diff --git a/app/jobs/promise_progress_updater_job.rb b/app/jobs/promise_progress_updater_job.rb new file mode 100644 index 0000000..254b2b7 --- /dev/null +++ b/app/jobs/promise_progress_updater_job.rb @@ -0,0 +1,7 @@ +class PromiseProgressUpdaterJob < ApplicationJob + queue_as :default + + def perform(promise) + promise.update_progress!(inline: true) + end +end diff --git a/app/models/evidence.rb b/app/models/evidence.rb index eb13f6c..138336f 100644 --- a/app/models/evidence.rb +++ b/app/models/evidence.rb @@ -4,6 +4,18 @@ class Evidence < ApplicationRecord belongs_to :linked_by, class_name: "User", optional: true belongs_to :reviewed_by, class_name: "User", optional: true + def format_for_llm + <<~XML + + #{activity.title} + #{activity.entry.feed} + #{linked_at} + #{impact_reason} + #{activity.entry.url} + + XML + end + after_commit do self.promise.set_last_evidence_date! end diff --git a/app/models/promise.rb b/app/models/promise.rb index 4f03fe5..d03a501 100644 --- a/app/models/promise.rb +++ b/app/models/promise.rb @@ -24,15 +24,30 @@ def link_department!(department, is_lead: false) dp.save! end + + def format_for_llm { promise_id: promise_id, title: concise_title, description: description, - text: text + text: text, + responsible_department_lead: responsible_department_lead, + intended_impact_and_objectives: intended_impact_and_objectives, + background_and_context: background_and_context } end + def update_progress!(inline: false) + unless inline + return PromiseProgressUpdaterJob.perform_later(self) + end + extractor = PromiseProgressUpdater.create!(record: self) + extractor.update_promise_progress! + self.save! + end + + def self.client_fields [ :id, diff --git a/app/models/promise_progress_updater.rb b/app/models/promise_progress_updater.rb new file mode 100644 index 0000000..bdfe1f2 --- /dev/null +++ b/app/models/promise_progress_updater.rb @@ -0,0 +1,131 @@ +class PromiseProgressUpdater < Chat + include Structify::Model + + def prompt(promise) + <<~PROMPT + You are a specialized government accountability analyst. Your task is to assess the progress made on specific government commitments based on available evidence of government actions. + + **Your Mission:** + Analyze the provided government commitment and associated evidence to determine how much progress has been made toward fulfilling that commitment. You will assign a progress score and provide a factual summary based solely on the evidence provided. + + **Input Data Structure:** + You will receive: + 1. **Promise Information:** + - `promise_id`: The internal ID for the promise + - `title`:#{' '} + - `description`: + - `text`:#{' '} + - `responsible_department_lead`: The department responsible for this commitment + - `intended_impact_and_objectives`: What the commitment aims to achieve + - `background_and_context`: Additional context about the commitment + + 2. **Evidence Items:** A list of government actions/evidence related to this commitment, each containing: + - `title_or_summary`: Brief description of the government action + - `evidence_source_type`: Type of evidence (e.g., "Bill Event (LEGISinfo)", "Canada Gazette Part II", "OIC", "News") + - `evidence_date`: When this action occurred (YYYY-MM-DD format) + - `description_or_details`: Detailed description of the action + - `source_url`: Official government source URL + - `bill_one_sentence_description_llm`: (For bills only) AI-generated description of the bill's purpose + + First, carefully read and analyze the following promise information: + + #{promise.format_for_llm} + + + Now, review the list of evidence items: + + + #{promise.evidences.map(&:format_for_llm).join("\n")} + + + **Progress Scoring Scale (1-5):** + + **Score 1 - No Progress:** + - No meaningful government action found + - No relevant legislation introduced + - No funding allocated or programs launched + + **Score 2 - Initial Steps:** + - Early-stage actions like consultations launched + - Preliminary announcements or studies initiated + - Minor policy discussions or planning activities + - No significant legislative action or substantial funding + + **Score 3 - Meaningful Action:** + - Legislation introduced and progressing through Parliament + - Significant budget allocation announced or programs launched + - Substantial policy development or regulatory work initiated + - Clear government commitment with concrete steps taken + + **Score 4 - Major Progress:** + - Key legislation passed major parliamentary stages (e.g., passed one House) + - Substantial regulatory changes enacted or published + - Significant funding disbursed and programs operational + - Major implementation milestones achieved + + **Score 5 - Complete/Fully Implemented:** + - All necessary legislation received Royal Assent and in force + - Key regulations published and operational + - All announced funding allocated and programs fully operational + - Commitment objectives substantially achieved + + **Analysis Guidelines:** + 1. **Evidence-Based Assessment:** Base your score only on the provided evidence + 2. **Legislative Tracking:** Consider all stages of bill progress (introduction, readings, committee, Royal Assent) + 3. **Implementation Focus:** Distinguish between announcements and actual implementation + 4. **Proportional Scoring:** Consider the scope and complexity of the commitment + 5. **Temporal Relevance:** Focus on actions within the current parliamentary session + + **Output Format:** + Provide your assessment as a JSON object with this exact structure: + + ```json + { + "progress_score": 3, + "progress_summary": "A concise, factual summary (max 150 words) describing the key actions taken and current status based on the evidence provided. Focus on concrete actions, legislative milestones, funding allocations, and implementation status." + } + ``` + + **Key Requirements:** + - **Objectivity:** Base assessments only on provided evidence, avoid speculation + - **Clarity:** Use clear, factual language in the progress summary + - **Completeness:** Consider all evidence items when determining the score + - **Accuracy:** Ensure the score aligns with the evidence and scoring criteria + - **Conciseness:** Keep the summary focused and under 150 words + + **Example Scoring Logic:** + - If a bill was introduced but hasn't progressed → Score 2-3 + - If a bill passed one House of Parliament → Score 3-4#{' '} + - If a bill received Royal Assent → Score 4-5 + - If funding was announced but not yet disbursed → Score 2-3 + - If programs are operational with funding flowing → Score 4-5 + - If no relevant evidence found → Score 1#{' '} + PROMPT + end + + + schema_definition do + version 1 + name "PromiseProgressUpdater" + description "Updates the progress of a promise" + field :promise_summary, :object, properties: { + "progress_score" => { type: "integer" }, + "progress_summary" => { type: "string", description: "Summary of the progress the government has made towards a promise" } + } + end + + + def update_promise_progress! + raise ArgumentError.new("Promise is not provided") unless self.record and self.record.is_a?(Promise) + + p = prompt( + self.record + ) + + self.extract! p + + self.record.progress_score = promise_summary["progress_score"] + self.record.progress_summary = promise_summary["progress_summary"] + self.record.save! + end +end From 1f585afc45226bdfa2d9a2e64835ebec72b55f86 Mon Sep 17 00:00:00 2001 From: xrendan Date: Tue, 8 Jul 2025 22:36:32 -0600 Subject: [PATCH 2/3] Update format_for_llm to remove dead fields --- app/models/promise.rb | 5 +---- app/models/promise_progress_updater.rb | 9 +++------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/app/models/promise.rb b/app/models/promise.rb index d03a501..2be5c0b 100644 --- a/app/models/promise.rb +++ b/app/models/promise.rb @@ -31,10 +31,7 @@ def format_for_llm promise_id: promise_id, title: concise_title, description: description, - text: text, - responsible_department_lead: responsible_department_lead, - intended_impact_and_objectives: intended_impact_and_objectives, - background_and_context: background_and_context + text: text } end diff --git a/app/models/promise_progress_updater.rb b/app/models/promise_progress_updater.rb index bdfe1f2..f172cdb 100644 --- a/app/models/promise_progress_updater.rb +++ b/app/models/promise_progress_updater.rb @@ -12,12 +12,9 @@ def prompt(promise) You will receive: 1. **Promise Information:** - `promise_id`: The internal ID for the promise - - `title`:#{' '} - - `description`: - - `text`:#{' '} - - `responsible_department_lead`: The department responsible for this commitment - - `intended_impact_and_objectives`: What the commitment aims to achieve - - `background_and_context`: Additional context about the commitment + - `title`: Title of the promise + - `description`: Summary of the promise + - `text`: Original text of the promise 2. **Evidence Items:** A list of government actions/evidence related to this commitment, each containing: - `title_or_summary`: Brief description of the government action From 3156ef4815450a57d283b8b29ea51d28aebc48f5 Mon Sep 17 00:00:00 2001 From: xrendan Date: Tue, 8 Jul 2025 22:38:32 -0600 Subject: [PATCH 3/3] lint --- app/models/evidence.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/evidence.rb b/app/models/evidence.rb index 0581e5b..5f81a7b 100644 --- a/app/models/evidence.rb +++ b/app/models/evidence.rb @@ -3,7 +3,7 @@ class Evidence < ApplicationRecord belongs_to :promise belongs_to :linked_by, class_name: "User", optional: true belongs_to :reviewed_by, class_name: "User", optional: true - + def self.ransackable_attributes(auth_object = nil) [ "impact", "impact_reason", "link_reason", "link_type" ] end @@ -23,7 +23,7 @@ def format_for_llm XML end - + after_commit do self.promise.set_last_evidence_date! end