diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b80ddec..1e0af0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,31 @@ jobs: - run: npm ci - run: npm run lint:tsc + # The standalone scripts carry no .py extension on purpose, so a + # directory scan would silently skip them. The shell expands the + # pdc-* globs below instead, so new scripts following the repo's + # pdc- naming convention are picked up without touching this file. + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - run: pip install ruff==0.17.4 + - run: ruff check standalone/pdc-* + - run: ruff format --check standalone/pdc-* + + bandit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - run: pip install bandit==1.9.4 + - run: bandit -q standalone/pdc-* + build: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 4361605..68bce43 100644 --- a/README.md +++ b/README.md @@ -48,3 +48,18 @@ The `.env` file and `config.*.json` files are ignored by git, as they likely contain credentials and should not be committed. Finally, every option can be specified on the command line itself. + +## Standalone Python scripts + +The Python scripts in `standalone/` carry no filename extension, so +a directory scan would silently skip them. CI lints them with ruff +and bandit by shell-expanding the `standalone/pdc-*` glob, so for a +new script to be covered by lint, its name should start with `pdc-`. + +To run the same checks locally: + +``` +ruff check standalone/pdc-* +ruff format --check standalone/pdc-* +bandit -q standalone/pdc-* +``` diff --git a/standalone/pdc-optimize-field-matching b/standalone/pdc-optimize-field-matching index 4737947..2546d59 100755 --- a/standalone/pdc-optimize-field-matching +++ b/standalone/pdc-optimize-field-matching @@ -138,6 +138,7 @@ def retrieve_pdc_data(endpoint: str, *, params: dict[str, Any] | None = None): f"{DS_PDC_API_BASE_URL}{endpoint}", headers={"Authorization": f"Bearer {access_token}"}, params=params or {}, + timeout=60, ) pdc_data_dict = pdc_data_response.json() @@ -231,7 +232,6 @@ class RAG(dspy.Module): ) - @app.command() def retrieve_application_forms(): retrieve_pdc_data( @@ -396,7 +396,7 @@ def optimize_rag(): num_threads=2, track_stats=True, use_merge=False, - reflection_lm=lm, # TODO: use a separate reflection LLM + reflection_lm=lm, # TODO: use a separate reflection LLM ) rag = RAG() @@ -414,7 +414,7 @@ def optimize_rag(): optimized_rag = optimizer.compile( rag, trainset=trainset_examples, - valset=trainset_examples, # TODO: use a separate validation set + valset=trainset_examples, # TODO: use a separate validation set ) optimized_rag.save(base_path / "optimized_rag.json") diff --git a/standalone/pdc-process-base-fields b/standalone/pdc-process-base-fields index 8486699..a9e6d5b 100755 --- a/standalone/pdc-process-base-fields +++ b/standalone/pdc-process-base-fields @@ -69,8 +69,8 @@ those columns can simply do so. } """ -import json import argparse +import json import sys try: @@ -107,14 +107,18 @@ def main(): # here would be RawDescriptionHelpFormatter -- I don't usually # use that because it can do some minor reformatting that I # find aesthetically displeasing, but hey, YMMV. - formatter_class=argparse.RawTextHelpFormatter) - allowed_formats = ["csv",] # maybe also "markdown", "html", etc some day + formatter_class=argparse.RawTextHelpFormatter, + ) + allowed_formats = [ + "csv", + ] # maybe also "markdown", "html", etc some day arg_parser.add_argument( - '--output-format', + "--output-format", choices=allowed_formats, type=str.lower, default="csv", - help="Output format to show base fields in. Default: 'csv'") + help="Output format to show base fields in. Default: 'csv'", + ) # Parse command line. args = arg_parser.parse_args() @@ -123,11 +127,12 @@ def main(): json_input = json.load(sys.stdin) parsed_input = pandas.json_normalize(json_input) # Switch to index=True to include a first column that shows DataFrame index - parsed_input.to_csv(sys.stdout, index=False, encoding='utf-8') + parsed_input.to_csv(sys.stdout, index=False, encoding="utf-8") else: # can't happen raise ValueError( - f"ERROR: Unknown output format 'f{args.output_format}' requested") + f"ERROR: Unknown output format 'f{args.output_format}' requested" + ) -if __name__ == '__main__': +if __name__ == "__main__": main()