未整理ファイルアップ

.github/workflows/pyruff.yml

@@ -0,0 +1,66 @@
+name: Python Lint with Ruff
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+      - develop
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - 'pyproject.toml'
+      - 'ruff.toml'
+      - 'requirements.txt'
+      - 'requirements-dev.txt'
+
+jobs:
+  python-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        id: installDependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+
+      - name: Check Initer
+        id: checkIniter
+        run: |
+          echo "Running Ruff Lint Check..."
+          python -m ruff check . --exit-zero --no-cache --output-format json --output-file ruff-report.json
+          echo "Ruff Lint Check completed. ruff-report.json"
+
+      - name: Generate Linter Report
+        id: generateLinterReport
+        run: |
+          python scripts/generate_linter.py
+
+
+      - name: pull_request message with Ruff Lint results
+        id: prMessageRuffLint
+        run: |
+          # イベントがプルリクエストの場合はPRにコメントを追加する
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            echo "Posting Ruff Lint results to Pull Request..."
+            curl -v -X POST \
+            -H "Content-Type: application/json" \
+            -H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \
+            -d @lint-result.json \
+            ${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}/issues/${{ github.event.pull_request.number }}/comments
+          else
+            echo "Not a pull request event."
+            echo "Ruff Lint results:"
+            echo "-------------------"
+            cat lint-result.md
+            echo "-------------------"
+            echo "No PR detected. Skipping API comment."
+          fi

.github/workflows/pytest.yml

@@ -0,0 +1,99 @@
+name: Python Test
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+      # - develop
+    paths:
+      - 'src/**'
+      - 'tests/**'
+      - '.github/workflows/pytest.yml'
+      - 'requirements.txt'
+      - 'requirements-dev.txt'
+jobs:
+  python-test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        id: installDependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+
+      - name: Run Python Test
+        id: runPyTest
+        run: |
+          pytest --junitxml=pytest.xml --cov-report term-missing --cov=src tests/ | tee pytest-coverage.txt
+
+      - name: Coverage Report
+        id: CoverageReport
+        if: success()  # テスト成功時のみ実行
+        run: |
+          coverage-badge -o .coverage.svg
+          python - <<EOF
+          from scripts.generate_coverage import GenerateCoverage
+          generate_coverage = GenerateCoverage()
+          generate_coverage.save_table()
+          EOF
+
+      - name: Generate coverage-report Branch AND README.md
+        id: generateCoverageReportBranch
+        if: success()  # テスト成功時のみ実行
+        run: |
+          #  coverage-report ブランチが存在しない場合は作成 あればチェックアウト
+          if git ls-remote --exit-code origin coverage-report; then
+            echo "coverage-report branch exists"
+            git fetch origin coverage-report:coverage-report
+            git checkout -B coverage-report origin/coverage-report          
+          else
+            echo "coverage-report branch does not exist"
+            git checkout --orphan coverage-report
+            git rm -rf .  # すべてのファイルを削除
+          fi
+          ls -l         
+
+
+
+      - name: Update Readme
+        id: updateReadme
+        run: |
+          ls -l         
+          mv .coverage.svg  coverage.svg
+          echo "# Pytest Report" > README.md
+          echo "" >> README.md
+          echo "![test](coverage.svg)" >> README.md
+          echo "" >> README.md
+          cat coverage_table.md >> README.md
+          cat README.md
+
+      - name: Check files before upload
+        id: checkFiles
+        run: ls -l README.md coverage.svg
+
+      - name: Commit Test Report To coverage-report Branch
+        id: commitTestReport
+        if: success()  # テスト成功時のみ実行
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+          git add README.md coverage.svg
+
+          # 変更があるかどうか確認（ステージング領域）
+          if git diff --cached --quiet; then
+            echo "No changes to commit"
+          else
+            git commit -m "Update coverage report"
+            git push https://actions-bot:${{ secrets.CICD_GITEA_TOKEN }}@gitea.pglikers.com/data-science/cloud-run-job-base.git coverage-report --force
+          fi
+

examples/dag.py

@@ -0,0 +1,17 @@
+"""Airflow DAGの定義ファイル"""
+from airflow import DAG
+from airflow.operators.bash import BashOperator
+from datetime import datetime
+
+with DAG(
+    dag_id="run_job_py",
+    start_date=datetime(2024, 1, 1),
+    schedule_interval="@daily",
+    catchup=False,
+) as dag:
+
+    run_job = BashOperator(
+        task_id="run_job",
+        bash_command="python main.py",
+    )
+

examples/pipeline.py

@@ -0,0 +1,18 @@
+"""jobを実行するためのエントリポイント
+
+Note: perfectのWorkflow Jobの場合はこちらから実行されます。
+"""
+from prefect import flow, task
+import jobs.job_example as job
+
+@task
+def _run_job():
+    job.main()
+
+
+@flow
+def _flow():
+    _run_job()
+
+if __name__ == "__main__":
+    _flow()

infra/k8s/clone_job.yaml

@@ -0,0 +1,21 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: wf-<proj>-<env>-<job>-<schedule>
+spec:
+  schedule: "0 * * * *"
+spec:
+  schedule: "0 * * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+            - name: main
+              image: alpine:3.20
+              command: ["sh","-lc"]
+              args: ["date; echo run"]

infra/k8s/job.yaml

@@ -0,0 +1,29 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+metadata:
+  generateName: job-<proj>-<env>-<job>-
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 120
+  template:
+    spec:
+      restartPolicy: Never
+      volumes:
+      - name: work
+        emptyDir: {}
+      initContainers:
+      - name: git-clone
+        image: alpine/git:2.45.2
+        args: ["clone","--depth=1","https://github.com/ORG/REPO.git","/work"]
+        volumeMounts:
+        - name: work
+          mountPath: /work
+      containers:
+      - name: run
+        image: python:3.12-slim
+        workingDir: /work
+        command: ["python","main.py"]
+        volumeMounts:
+        - name: work
+          mountPath: /work

infra/k8s/workflow.yaml

@@ -0,0 +1,12 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: wf-<proj>-<env>-<job>-
+spec:
+  entrypoint: run
+  templates:
+  - name: run
+    volumes:
+    - name: work
+      emptyDir: {}
+

pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "プロジェクト名を設定してください"
+version = "0.1.0"
+description = "プロジェクトの説明"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = []

reademe/job.md

@@ -0,0 +1,28 @@
+
+# JOBについて
+
+## Jobを実行する環境
+
+JOBで起動する場合には様々なデプロイ方法があります。
+単発で実行するする場合と、DAGによる管理が必要な場合があります。
+
+| ツール名       | タイプ   | 実行する場所          |
+| -------------- | -------- | --------------------- |
+| Cloud Runs JOB | JOB      | GCP                   |
+| K8s JOB        | JOB      | K8s                   |
+| perfect        | Workflow | VM/オンプレ/Cloud Run |
+| Argo Workflow  | Workflow | K8s                   |
+| Apache Airflow | Workflow | GCP/VM/オンプレ       |
+
+* Perfectは2レイヤ構造になっている
+  * Server　Sassを使うと「VM 1台 + Cloud Run Jobs」、
+  * Prefect Server/Cloud: UI・スケジューラ・状態管理（VM/オンプレ）
+  * Worker: 実行指示・起動: VM/オンプレ(Workerは常時起動が必要)
+  * Job実行体: 実際処理 -> Cloud Run Jobs
+* Cloud Composer(GCPのAirflow) + Cloud Runs JOB
+  * クラウドサービスでの王道パターンです
+  * Cloud Runs JOBで構築するなら一番良さそうです
+* K8sで実行するなArgo Workflowの一択
+  * Argo Workflow + Cloud Runs JOB
+  * Argo WorkflowでPodを実行する
+

src/jobs/job_example.py

@@ -0,0 +1,16 @@
+
+
+# job.py
+def run(params: dict | None = None):
+    print("business logic")
+    print(params)
+
+
+def main():
+    # CLI / subprocess 用の入口
+    params = {"env": "dev"}
+    run(params)
+
+
+if __name__ == "__main__":
+    main()

src/main.py

@@ -0,0 +1,11 @@
+"""jobを実行するためのエントリポイント
+
+Note: Cloud Run Jobの場合はこちらから実行されます。
+"""
+import jobs.job_example
+
+def main():
+    jobs.job_example.main()
+
+if __name__ == "__main__":
+    main()