diff --git a/api/api.py b/api/api.py index d40e73f96..13df30c32 100644 --- a/api/api.py +++ b/api/api.py @@ -60,7 +60,7 @@ class ProcessedProjectEntry(BaseModel): class RepoInfo(BaseModel): owner: str repo: str - type: str + type: str # Repository type: github, gitlab, bitbucket, azure_devops token: Optional[str] = None localPath: Optional[str] = None repoUrl: Optional[str] = None diff --git a/api/config/generator.json b/api/config/generator.json index f88179098..6560e1eb2 100644 --- a/api/config/generator.json +++ b/api/config/generator.json @@ -37,6 +37,11 @@ "temperature": 1.0, "top_p": 0.8, "top_k": 20 + }, + "gemini-3.1-flash-lite-preview": { + "temperature": 1.0, + "top_p": 0.8, + "top_k": 20 } } }, diff --git a/api/data_pipeline.py b/api/data_pipeline.py index 5e1f5fa47..b67616daf 100644 --- a/api/data_pipeline.py +++ b/api/data_pipeline.py @@ -118,6 +118,11 @@ def download_repo(repo_url: str, local_path: str, repo_type: str = None, access_ elif repo_type == "bitbucket": # Format: https://x-token-auth:{token}@bitbucket.org/owner/repo.git clone_url = urlunparse((parsed.scheme, f"x-token-auth:{encoded_token}@{parsed.netloc}", parsed.path, '', '', '')) + elif repo_type == "azure_devops": + # Format: https://{token}@dev.azure.com/org/project/_git/repo + # Strip any existing username from netloc (ADO URLs often include user@) + hostname = parsed.hostname or parsed.netloc.split('@')[-1] + clone_url = urlunparse((parsed.scheme, f"{encoded_token}@{hostname}", parsed.path, '', '', '')) logger.info("Using access token for authentication") @@ -684,6 +689,59 @@ def get_bitbucket_file_content(repo_url: str, file_path: str, access_token: str raise ValueError(f"Failed to get file content: {str(e)}") +def get_azure_devops_file_content(repo_url: str, file_path: str, access_token: str = None) -> str: + """ + Retrieves the content of a file from an Azure DevOps repository using the ADO REST API. + + Args: + repo_url (str): The URL (e.g., "https://dev.azure.com/org/project/_git/repo") + file_path (str): Path to file (e.g., "src/main.py") + access_token (str, optional): Azure DevOps PAT + + Returns: + str: The content of the file + """ + try: + parsed_url = urlparse(repo_url) + if not parsed_url.scheme or not parsed_url.netloc: + raise ValueError("Not a valid Azure DevOps repository URL") + + path_parts = parsed_url.path.strip('/').split('/') + + if '_git' not in path_parts: + raise ValueError("Not a valid Azure DevOps repository URL - missing _git in path") + + git_index = path_parts.index('_git') + repo_name = path_parts[git_index + 1].replace(".git", "") if git_index + 1 < len(path_parts) else None + + if not repo_name: + raise ValueError("Could not extract repository name from Azure DevOps URL") + + # Build API URL: https://dev.azure.com/{org}/{project}/_apis/git/repositories/{repo}/items + # Strip any existing username from netloc (ADO URLs often include user@) + hostname = parsed_url.hostname or parsed_url.netloc.split('@')[-1] + project_path = '/'.join(path_parts[:git_index]) + api_base = f"{parsed_url.scheme}://{hostname}/{project_path}" + api_url = f"{api_base}/_apis/git/repositories/{repo_name}/items?path={file_path}&api-version=7.0" + + headers = {} + if access_token: + encoded = base64.b64encode(f":{access_token}".encode()).decode() + headers["Authorization"] = f"Basic {encoded}" + + logger.info(f"Fetching file content from Azure DevOps API: {api_url}") + try: + response = requests.get(api_url, headers=headers) + response.raise_for_status() + except RequestException as e: + raise ValueError(f"Error fetching file content: {e}") + + return response.text + + except Exception as e: + raise ValueError(f"Failed to get file content: {str(e)}") + + def get_file_content(repo_url: str, file_path: str, repo_type: str = None, access_token: str = None) -> str: """ Retrieves the content of a file from a Git repository (GitHub or GitLab). @@ -706,8 +764,10 @@ def get_file_content(repo_url: str, file_path: str, repo_type: str = None, acces return get_gitlab_file_content(repo_url, file_path, access_token) elif repo_type == "bitbucket": return get_bitbucket_file_content(repo_url, file_path, access_token) + elif repo_type == "azure_devops": + return get_azure_devops_file_content(repo_url, file_path, access_token) else: - raise ValueError("Unsupported repository type. Only GitHub, GitLab, and Bitbucket are supported.") + raise ValueError("Unsupported repository type. Only GitHub, GitLab, Bitbucket, and Azure DevOps are supported.") class DatabaseManager: """ @@ -763,7 +823,17 @@ def _extract_repo_name_from_url(self, repo_url_or_path: str, repo_type: str) -> # Extract owner and repo name to create unique identifier url_parts = repo_url_or_path.rstrip('/').split('/') - if repo_type in ["github", "gitlab", "bitbucket"] and len(url_parts) >= 5: + if repo_type == "azure_devops": + # Azure DevOps URL: https://dev.azure.com/{org}/{project}/_git/{repo} + try: + git_index = url_parts.index('_git') + repo = url_parts[git_index + 1].replace(".git", "") + project = url_parts[git_index - 1] + repo_name = f"{project}_{repo}" + except (ValueError, IndexError): + repo_name = url_parts[-1].replace(".git", "") + return repo_name + elif repo_type in ["github", "gitlab", "bitbucket"] and len(url_parts) >= 5: # GitHub URL format: https://github.com/owner/repo # GitLab URL format: https://gitlab.com/owner/repo or https://gitlab.com/group/subgroup/repo # Bitbucket URL format: https://bitbucket.org/owner/repo diff --git a/api/simple_chat.py b/api/simple_chat.py index 41a184ed8..35a035568 100644 --- a/api/simple_chat.py +++ b/api/simple_chat.py @@ -61,7 +61,7 @@ class ChatCompletionRequest(BaseModel): messages: List[ChatMessage] = Field(..., description="List of chat messages") filePath: Optional[str] = Field(None, description="Optional path to a file in the repository to include in the prompt") token: Optional[str] = Field(None, description="Personal access token for private repositories") - type: Optional[str] = Field("github", description="Type of repository (e.g., 'github', 'gitlab', 'bitbucket')") + type: Optional[str] = Field("github", description="Type of repository (e.g., 'github', 'gitlab', 'bitbucket', 'azure_devops')") # model parameters provider: str = Field("google", description="Model provider (google, openai, openrouter, ollama, bedrock, azure, dashscope)") diff --git a/api/websocket_wiki.py b/api/websocket_wiki.py index d1a6b1bd5..829004fd1 100644 --- a/api/websocket_wiki.py +++ b/api/websocket_wiki.py @@ -45,7 +45,7 @@ class ChatCompletionRequest(BaseModel): messages: List[ChatMessage] = Field(..., description="List of chat messages") filePath: Optional[str] = Field(None, description="Optional path to a file in the repository to include in the prompt") token: Optional[str] = Field(None, description="Personal access token for private repositories") - type: Optional[str] = Field("github", description="Type of repository (e.g., 'github', 'gitlab', 'bitbucket')") + type: Optional[str] = Field("github", description="Type of repository (e.g., 'github', 'gitlab', 'bitbucket', 'azure_devops')") # model parameters provider: str = Field( diff --git a/src/app/[owner]/[repo]/page.tsx b/src/app/[owner]/[repo]/page.tsx index 98ffc6fbc..c78638980 100644 --- a/src/app/[owner]/[repo]/page.tsx +++ b/src/app/[owner]/[repo]/page.tsx @@ -13,7 +13,7 @@ import { extractUrlDomain, extractUrlPath } from '@/utils/urlDecoder'; import Link from 'next/link'; import { useParams, useSearchParams } from 'next/navigation'; import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { FaBitbucket, FaBookOpen, FaComments, FaDownload, FaExclamationTriangle, FaFileExport, FaFolder, FaGithub, FaGitlab, FaHome, FaSync, FaTimes } from 'react-icons/fa'; +import { FaBitbucket, FaBookOpen, FaComments, FaDownload, FaExclamationTriangle, FaFileExport, FaFolder, FaGithub, FaGitlab, FaHome, FaMicrosoft, FaSync, FaTimes } from 'react-icons/fa'; // Define the WikiSection and WikiStructure types directly in this file // since the imported types don't have the sections and rootSections properties interface WikiSection { @@ -173,6 +173,18 @@ const createBitbucketHeaders = (bitbucketToken: string): HeadersInit => { return headers; }; +const createAzureDevOpsHeaders = (adoToken: string): HeadersInit => { + const headers: HeadersInit = { + 'Content-Type': 'application/json', + }; + + if (adoToken) { + headers['Authorization'] = `Basic ${btoa(':' + adoToken)}`; + } + + return headers; +}; + export default function RepoWikiPage() { // Get route parameters and search params @@ -205,9 +217,11 @@ export default function RepoWikiPage() { ? 'bitbucket' : repoHost?.includes('gitlab') ? 'gitlab' - : repoHost?.includes('github') - ? 'github' - : searchParams.get('type') || 'github'; + : repoHost?.includes('dev.azure.com') + ? 'azure_devops' + : repoHost?.includes('github') + ? 'github' + : searchParams.get('type') || 'github'; // Import language context for translations const { messages } = useLanguage(); @@ -1479,6 +1493,68 @@ IMPORTANT: console.warn('Could not fetch Bitbucket README.md, continuing with empty README', err); } } + else if (effectiveRepoInfo.type === 'azure_devops') { + // Azure DevOps API approach + const adoUrl = effectiveRepoInfo.repoUrl ?? ''; + const adoParsed = new URL(adoUrl); + const adoParts = adoParsed.pathname.replace(/^\/|\/$/g, '').split('/'); + const gitIndex = adoParts.indexOf('_git'); + + if (gitIndex < 1 || gitIndex + 1 >= adoParts.length) { + throw new Error('Invalid Azure DevOps repository URL'); + } + + const adoRepo = adoParts[gitIndex + 1]; + const adoBase = `${adoParsed.protocol}//${adoParsed.hostname}/${adoParts.slice(0, gitIndex).join('/')}`; + const headers = createAzureDevOpsHeaders(currentToken); + + // Get default branch + let defaultBranchLocal = 'main'; + try { + const repoInfoUrl = `${adoBase}/_apis/git/repositories/${adoRepo}?api-version=7.0`; + const repoInfoRes = await fetch(repoInfoUrl, { headers }); + if (repoInfoRes.ok) { + const repoData = await repoInfoRes.json(); + // ADO returns defaultBranch as "refs/heads/main" + defaultBranchLocal = (repoData.defaultBranch || 'refs/heads/main').replace('refs/heads/', ''); + console.log(`Found Azure DevOps default branch: ${defaultBranchLocal}`); + } + } catch (err) { + console.warn('Could not fetch ADO repo info:', err); + } + setDefaultBranch(defaultBranchLocal); + + // Get file tree using Items API with recursion + const treeUrl = `${adoBase}/_apis/git/repositories/${adoRepo}/items?recursionLevel=Full&versionDescriptor.version=${defaultBranchLocal}&api-version=7.0`; + const treeRes = await fetch(treeUrl, { headers }); + + if (!treeRes.ok) { + const errorData = await treeRes.text(); + throw new Error(`Azure DevOps API error: ${treeRes.status} - ${errorData}`); + } + + const treeData = await treeRes.json(); + + if (!treeData.value || treeData.value.length === 0) { + throw new Error('Could not fetch repository structure from Azure DevOps.'); + } + + fileTreeData = treeData.value + .filter((item: { gitObjectType: string; path: string }) => item.gitObjectType === 'blob') + .map((item: { path: string }) => item.path.replace(/^\//, '')) + .join('\n'); + + // Fetch README + try { + const readmeUrl = `${adoBase}/_apis/git/repositories/${adoRepo}/items?path=README.md&api-version=7.0`; + const readmeRes = await fetch(readmeUrl, { headers }); + if (readmeRes.ok) { + readmeContent = await readmeRes.text(); + } + } catch (err) { + console.warn('Could not fetch Azure DevOps README.md:', err); + } + } // Now determine the wiki structure await determineWikiStructure(fileTreeData, readmeContent, owner, repo); @@ -2059,6 +2135,8 @@ IMPORTANT: ) : effectiveRepoInfo.type === 'gitlab' ? ( + ) : effectiveRepoInfo.type === 'azure_devops' ? ( + ) : ( )} @@ -2269,7 +2347,7 @@ IMPORTANT: onApply={confirmRefresh} showWikiType={true} showTokenInput={effectiveRepoInfo.type !== 'local' && !currentToken} // Show token input if not local and no current token - repositoryType={effectiveRepoInfo.type as 'github' | 'gitlab' | 'bitbucket'} + repositoryType={effectiveRepoInfo.type as 'github' | 'gitlab' | 'bitbucket' | 'azure_devops'} authRequired={authRequired} authCode={authCode} setAuthCode={setAuthCode} diff --git a/src/app/page.tsx b/src/app/page.tsx index 9e05a2ef9..7d6f2968f 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -134,7 +134,7 @@ export default function Home() { const [excludedFiles, setExcludedFiles] = useState(''); const [includedDirs, setIncludedDirs] = useState(''); const [includedFiles, setIncludedFiles] = useState(''); - const [selectedPlatform, setSelectedPlatform] = useState<'github' | 'gitlab' | 'bitbucket'>('github'); + const [selectedPlatform, setSelectedPlatform] = useState<'github' | 'gitlab' | 'bitbucket' | 'azure_devops'>('github'); const [accessToken, setAccessToken] = useState(''); const [error, setError] = useState(null); const [isSubmitting, setIsSubmitting] = useState(false); @@ -212,13 +212,23 @@ export default function Home() { type = 'gitlab'; } else if (domain?.includes('bitbucket.org') || domain?.includes('bitbucket.')) { type = 'bitbucket'; + } else if (domain?.includes('dev.azure.com')) { + type = 'azure_devops'; } else { type = 'web'; // fallback for other git hosting services } fullPath = extractUrlPath(input)?.replace(/\.git$/, ''); const parts = fullPath?.split('/') ?? []; - if (parts.length >= 2) { + + if (type === 'azure_devops') { + // ADO URL path: {org}/{project}/_git/{repo} + const gitIndex = parts.indexOf('_git'); + if (gitIndex >= 1 && gitIndex + 1 < parts.length) { + owner = parts[gitIndex - 1]; // project name + repo = parts[gitIndex + 1]; // repo name + } + } else if (parts.length >= 2) { repo = parts[parts.length - 1] || ''; owner = parts[parts.length - 2] || ''; } @@ -559,6 +569,10 @@ export default function Home() { className="bg-[var(--background)]/70 p-3 rounded border border-[var(--border-color)] font-mono overflow-x-hidden whitespace-nowrap" >https://bitbucket.org/atlassian/atlaskit + https://dev.azure.com/org/project/_git/repo + diff --git a/src/components/ConfigurationModal.tsx b/src/components/ConfigurationModal.tsx index 7a1dae697..0dc646967 100644 --- a/src/components/ConfigurationModal.tsx +++ b/src/components/ConfigurationModal.tsx @@ -32,8 +32,8 @@ interface ConfigurationModalProps { setCustomModel: (value: string) => void; // Platform selection - selectedPlatform: 'github' | 'gitlab' | 'bitbucket'; - setSelectedPlatform: (value: 'github' | 'gitlab' | 'bitbucket') => void; + selectedPlatform: 'github' | 'gitlab' | 'bitbucket' | 'azure_devops'; + setSelectedPlatform: (value: 'github' | 'gitlab' | 'bitbucket' | 'azure_devops') => void; // Access token accessToken: string; @@ -98,8 +98,8 @@ export default function ConfigurationModal({ }: ConfigurationModalProps) { const { messages: t } = useLanguage(); - // Show token section state - const [showTokenSection, setShowTokenSection] = useState(false); + // Show token section state - auto-expand for Azure DevOps since PAT is required + const [showTokenSection, setShowTokenSection] = useState(selectedPlatform === 'azure_devops'); if (!isOpen) return null; @@ -231,7 +231,34 @@ export default function ConfigurationModal({ /> - {/* Access token section using TokenInput component */} + {/* Platform selection - always visible */} + + + {t.form?.selectPlatform || 'Select Platform'} + + + {(['github', 'gitlab', 'bitbucket', 'azure_devops'] as const).map((platform) => ( + { + setSelectedPlatform(platform); + if (platform === 'azure_devops') setShowTokenSection(true); + }} + className={`flex-1 flex items-center justify-center gap-2 px-3 py-2 rounded-md border transition-all ${selectedPlatform === platform + ? 'bg-[var(--accent-primary)]/10 border-[var(--accent-primary)] text-[var(--accent-primary)] shadow-sm' + : 'border-[var(--border-color)] text-[var(--foreground)] hover:bg-[var(--background)]' + }`} + > + + {platform === 'azure_devops' ? 'Azure DevOps' : platform.charAt(0).toUpperCase() + platform.slice(1)} + + + ))} + + + + {/* Access token input */} setShowTokenSection(!showTokenSection)} - allowPlatformChange={true} + allowPlatformChange={false} /> {/* Authorization Code Input */} diff --git a/src/components/ModelSelectionModal.tsx b/src/components/ModelSelectionModal.tsx index 5a8ed3fe4..f09589221 100644 --- a/src/components/ModelSelectionModal.tsx +++ b/src/components/ModelSelectionModal.tsx @@ -37,7 +37,7 @@ interface ModelSelectionModalProps { // Token input for refresh showTokenInput?: boolean; - repositoryType?: 'github' | 'gitlab' | 'bitbucket'; + repositoryType?: 'github' | 'gitlab' | 'bitbucket' | 'azure_devops'; // Authentication authRequired?: boolean; authCode?: string; @@ -91,7 +91,7 @@ export default function ModelSelectionModal({ // Token input state const [localAccessToken, setLocalAccessToken] = useState(''); - const [localSelectedPlatform, setLocalSelectedPlatform] = useState<'github' | 'gitlab' | 'bitbucket'>(repositoryType); + const [localSelectedPlatform, setLocalSelectedPlatform] = useState<'github' | 'gitlab' | 'bitbucket' | 'azure_devops'>(repositoryType); const [showTokenSection, setShowTokenSection] = useState(showTokenInput); // Reset local state when modal is opened diff --git a/src/components/TokenInput.tsx b/src/components/TokenInput.tsx index 14fadcd8d..1bd18b330 100644 --- a/src/components/TokenInput.tsx +++ b/src/components/TokenInput.tsx @@ -4,8 +4,8 @@ import React from 'react'; import { useLanguage } from '@/contexts/LanguageContext'; interface TokenInputProps { - selectedPlatform: 'github' | 'gitlab' | 'bitbucket'; - setSelectedPlatform: (value: 'github' | 'gitlab' | 'bitbucket') => void; + selectedPlatform: 'github' | 'gitlab' | 'bitbucket' | 'azure_devops'; + setSelectedPlatform: (value: 'github' | 'gitlab' | 'bitbucket' | 'azure_devops') => void; accessToken: string; setAccessToken: (value: string) => void; showTokenSection?: boolean; @@ -24,7 +24,7 @@ export default function TokenInput({ }: TokenInputProps) { const { messages: t } = useLanguage(); - const platformName = selectedPlatform.charAt(0).toUpperCase() + selectedPlatform.slice(1); + const platformName = selectedPlatform === 'azure_devops' ? 'Azure DevOps' : selectedPlatform.charAt(0).toUpperCase() + selectedPlatform.slice(1); return ( @@ -76,6 +76,16 @@ export default function TokenInput({ > Bitbucket + setSelectedPlatform('azure_devops')} + className={`flex-1 flex items-center justify-center gap-2 px-3 py-2 rounded-md border transition-all ${selectedPlatform === 'azure_devops' + ? 'bg-[var(--accent-primary)]/10 border-[var(--accent-primary)] text-[var(--accent-primary)] shadow-sm' + : 'border-[var(--border-color)] text-[var(--foreground)] hover:bg-[var(--background)]' + }`} + > + Azure DevOps + )} diff --git a/test/test_azure_devops_file_content.py b/test/test_azure_devops_file_content.py new file mode 100644 index 000000000..0f76c6ee0 --- /dev/null +++ b/test/test_azure_devops_file_content.py @@ -0,0 +1,70 @@ +import pytest +import os +import sys +from unittest.mock import patch, Mock + +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) + +from api.data_pipeline import get_azure_devops_file_content, get_file_content + + +class TestAzureDevOpsFileContent: + + def test_get_file_content_dispatches_to_azure_devops(self): + with patch('api.data_pipeline.get_azure_devops_file_content') as mock_ado: + mock_ado.return_value = "file content" + result = get_file_content( + "https://dev.azure.com/org/project/_git/repo", + "src/main.py", + repo_type="azure_devops", + access_token="test-token" + ) + mock_ado.assert_called_once_with( + "https://dev.azure.com/org/project/_git/repo", + "src/main.py", + "test-token" + ) + assert result == "file content" + + @patch('api.data_pipeline.requests.get') + def test_get_azure_devops_file_content_modern_url(self, mock_get): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = "print('hello')" + mock_response.raise_for_status = Mock() + mock_get.return_value = mock_response + + result = get_azure_devops_file_content( + "https://dev.azure.com/myorg/myproject/_git/myrepo", + "src/main.py", + "my-pat-token" + ) + assert result == "print('hello')" + + call_url = mock_get.call_args[0][0] + assert "dev.azure.com/myorg/myproject/_apis/git/repositories/myrepo/items" in call_url + assert "path=src/main.py" in call_url + + def test_get_azure_devops_file_content_invalid_url(self): + with pytest.raises(ValueError, match="Not a valid Azure DevOps"): + get_azure_devops_file_content( + "https://github.com/owner/repo", + "file.py" + ) + + @patch('api.data_pipeline.requests.get') + def test_get_azure_devops_file_content_uses_basic_auth(self, mock_get): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.text = "content" + mock_response.raise_for_status = Mock() + mock_get.return_value = mock_response + + get_azure_devops_file_content( + "https://dev.azure.com/org/project/_git/repo", + "file.py", + "my-token" + ) + + actual_headers = mock_get.call_args + assert "Basic" in str(actual_headers) diff --git a/test/test_extract_repo_name.py b/test/test_extract_repo_name.py index 65a15da09..a32d4729c 100644 --- a/test/test_extract_repo_name.py +++ b/test/test_extract_repo_name.py @@ -67,6 +67,25 @@ def test_extract_repo_name_bitbucket_urls(self): print("✓ Bitbucket URL tests passed") + def test_extract_repo_name_azure_devops_urls(self): + """Test repository name extraction from Azure DevOps URLs""" + # Modern ADO URL + ado_url = "https://dev.azure.com/myorg/myproject/_git/myrepo" + result = self.db_manager._extract_repo_name_from_url(ado_url, "azure_devops") + assert result == "myproject_myrepo" + + # ADO URL with .git suffix + ado_git = "https://dev.azure.com/myorg/myproject/_git/myrepo.git" + result = self.db_manager._extract_repo_name_from_url(ado_git, "azure_devops") + assert result == "myproject_myrepo" + + # ADO URL with trailing slash + ado_slash = "https://dev.azure.com/myorg/myproject/_git/myrepo/" + result = self.db_manager._extract_repo_name_from_url(ado_slash, "azure_devops") + assert result == "myproject_myrepo" + + print("✓ Azure DevOps URL tests passed") + def test_extract_repo_name_local_paths(self): """Test repository name extraction from local paths""" result = self.db_manager._extract_repo_name_from_url("/home/user/projects/my-repo", "local")