From e05f9fc4bea1cf9d2a679f89f4e9ed222ca91c48 Mon Sep 17 00:00:00 2001 From: "ry.yamafuji" Date: Mon, 22 Dec 2025 00:26:42 +0900 Subject: [PATCH] =?UTF-8?q?=E6=9C=AA=E6=95=B4=E7=90=86=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=82=A2=E3=83=83=E3=83=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pyruff.yml | 66 ++++++++++++++++++++++++ .github/workflows/pytest.yml | 99 ++++++++++++++++++++++++++++++++++++ examples/dag.py | 17 +++++++ examples/pipeline.py | 18 +++++++ infra/k8s/clone_job.yaml | 21 ++++++++ infra/k8s/job.yaml | 29 +++++++++++ infra/k8s/workflow.yaml | 12 +++++ pyproject.toml | 7 +++ reademe/job.md | 28 ++++++++++ requirements.txt | 0 src/jobs/__init__.py | 0 src/jobs/job_example.py | 16 ++++++ src/main.py | 11 ++++ src/utils/cusom_logger.py | 0 14 files changed, 324 insertions(+) create mode 100644 .github/workflows/pyruff.yml create mode 100644 .github/workflows/pytest.yml create mode 100644 examples/dag.py create mode 100644 examples/pipeline.py create mode 100644 infra/k8s/clone_job.yaml create mode 100644 infra/k8s/job.yaml create mode 100644 infra/k8s/workflow.yaml create mode 100644 pyproject.toml create mode 100644 reademe/job.md create mode 100644 requirements.txt create mode 100644 src/jobs/__init__.py create mode 100644 src/jobs/job_example.py create mode 100644 src/main.py create mode 100644 src/utils/cusom_logger.py diff --git a/.github/workflows/pyruff.yml b/.github/workflows/pyruff.yml new file mode 100644 index 0000000..0945342 --- /dev/null +++ b/.github/workflows/pyruff.yml @@ -0,0 +1,66 @@ +name: Python Lint with Ruff + +on: + workflow_dispatch: + pull_request: + branches: + - main + - develop + paths: + - 'src/**' + - 'tests/**' + - 'pyproject.toml' + - 'ruff.toml' + - 'requirements.txt' + - 'requirements-dev.txt' + +jobs: + python-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install dependencies + id: installDependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Check Initer + id: checkIniter + run: | + echo "Running Ruff Lint Check..." + python -m ruff check . --exit-zero --no-cache --output-format json --output-file ruff-report.json + echo "Ruff Lint Check completed. ruff-report.json" + + - name: Generate Linter Report + id: generateLinterReport + run: | + python scripts/generate_linter.py + + + - name: pull_request message with Ruff Lint results + id: prMessageRuffLint + run: | + # イベントがプルリクエストの場合はPRにコメントを追加する + if [ "${{ github.event_name }}" = "pull_request" ]; then + echo "Posting Ruff Lint results to Pull Request..." + curl -v -X POST \ + -H "Content-Type: application/json" \ + -H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \ + -d @lint-result.json \ + ${{ gitea.server_url }}/api/v1/repos/${{ gitea.repository }}/issues/${{ github.event.pull_request.number }}/comments + else + echo "Not a pull request event." + echo "Ruff Lint results:" + echo "-------------------" + cat lint-result.md + echo "-------------------" + echo "No PR detected. Skipping API comment." + fi diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..113ddea --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,99 @@ +name: Python Test + +on: + workflow_dispatch: + push: + branches: + - main + # - develop + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/pytest.yml' + - 'requirements.txt' + - 'requirements-dev.txt' +jobs: + python-test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install dependencies + id: installDependencies + run: | + pip install -r requirements.txt + pip install -r requirements-dev.txt + + - name: Run Python Test + id: runPyTest + run: | + pytest --junitxml=pytest.xml --cov-report term-missing --cov=src tests/ | tee pytest-coverage.txt + + - name: Coverage Report + id: CoverageReport + if: success() # テスト成功時のみ実行 + run: | + coverage-badge -o .coverage.svg + python - < README.md + echo "" >> README.md + echo "![test](coverage.svg)" >> README.md + echo "" >> README.md + cat coverage_table.md >> README.md + cat README.md + + - name: Check files before upload + id: checkFiles + run: ls -l README.md coverage.svg + + - name: Commit Test Report To coverage-report Branch + id: commitTestReport + if: success() # テスト成功時のみ実行 + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" + + git add README.md coverage.svg + + # 変更があるかどうか確認（ステージング領域） + if git diff --cached --quiet; then + echo "No changes to commit" + else + git commit -m "Update coverage report" + git push https://actions-bot:${{ secrets.CICD_GITEA_TOKEN }}@gitea.pglikers.com/data-science/cloud-run-job-base.git coverage-report --force + fi + diff --git a/examples/dag.py b/examples/dag.py new file mode 100644 index 0000000..9f23ce4 --- /dev/null +++ b/examples/dag.py @@ -0,0 +1,17 @@ +"""Airflow DAGの定義ファイル""" +from airflow import DAG +from airflow.operators.bash import BashOperator +from datetime import datetime + +with DAG( + dag_id="run_job_py", + start_date=datetime(2024, 1, 1), + schedule_interval="@daily", + catchup=False, +) as dag: + + run_job = BashOperator( + task_id="run_job", + bash_command="python main.py", + ) + diff --git a/examples/pipeline.py b/examples/pipeline.py new file mode 100644 index 0000000..b77d50f --- /dev/null +++ b/examples/pipeline.py @@ -0,0 +1,18 @@ +"""jobを実行するためのエントリポイント + +Note: perfectのWorkflow Jobの場合はこちらから実行されます。 +""" +from prefect import flow, task +import jobs.job_example as job + +@task +def _run_job(): + job.main() + + +@flow +def _flow(): + _run_job() + +if __name__ == "__main__": + _flow() \ No newline at end of file diff --git a/infra/k8s/clone_job.yaml b/infra/k8s/clone_job.yaml new file mode 100644 index 0000000..50a1351 --- /dev/null +++ b/infra/k8s/clone_job.yaml @@ -0,0 +1,21 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: wf---- +spec: + schedule: "0 * * * *" +spec: + schedule: "0 * * * *" + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: Never + containers: + - name: main + image: alpine:3.20 + command: ["sh","-lc"] + args: ["date; echo run"] \ No newline at end of file diff --git a/infra/k8s/job.yaml b/infra/k8s/job.yaml new file mode 100644 index 0000000..d90b4e3 --- /dev/null +++ b/infra/k8s/job.yaml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: +metadata: + generateName: job---- +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 120 + template: + spec: + restartPolicy: Never + volumes: + - name: work + emptyDir: {} + initContainers: + - name: git-clone + image: alpine/git:2.45.2 + args: ["clone","--depth=1","https://github.com/ORG/REPO.git","/work"] + volumeMounts: + - name: work + mountPath: /work + containers: + - name: run + image: python:3.12-slim + workingDir: /work + command: ["python","main.py"] + volumeMounts: + - name: work + mountPath: /work diff --git a/infra/k8s/workflow.yaml b/infra/k8s/workflow.yaml new file mode 100644 index 0000000..7385c9c --- /dev/null +++ b/infra/k8s/workflow.yaml @@ -0,0 +1,12 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: wf---- +spec: + entrypoint: run + templates: + - name: run + volumes: + - name: work + emptyDir: {} + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..797506b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "プロジェクト名を設定してください" +version = "0.1.0" +description = "プロジェクトの説明" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [] \ No newline at end of file diff --git a/reademe/job.md b/reademe/job.md new file mode 100644 index 0000000..a91fc29 --- /dev/null +++ b/reademe/job.md @@ -0,0 +1,28 @@ + +# JOBについて + +## Jobを実行する環境 + +JOBで起動する場合には様々なデプロイ方法があります。 +単発で実行するする場合と、DAGによる管理が必要な場合があります。 + +| ツール名 | タイプ | 実行する場所 | +| -------------- | -------- | --------------------- | +| Cloud Runs JOB | JOB | GCP | +| K8s JOB | JOB | K8s | +| perfect | Workflow | VM/オンプレ/Cloud Run | +| Argo Workflow | Workflow | K8s | +| Apache Airflow | Workflow | GCP/VM/オンプレ | + +* Perfectは2レイヤ構造になっている + * Server　Sassを使うと「VM 1台 + Cloud Run Jobs」、 + * Prefect Server/Cloud: UI・スケジューラ・状態管理（VM/オンプレ） + * Worker: 実行指示・起動: VM/オンプレ(Workerは常時起動が必要) + * Job実行体: 実際処理 -> Cloud Run Jobs +* Cloud Composer(GCPのAirflow) + Cloud Runs JOB + * クラウドサービスでの王道パターンです + * Cloud Runs JOBで構築するなら一番良さそうです +* K8sで実行するなArgo Workflowの一択 + * Argo Workflow + Cloud Runs JOB + * Argo WorkflowでPodを実行する + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/src/jobs/__init__.py b/src/jobs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/jobs/job_example.py b/src/jobs/job_example.py new file mode 100644 index 0000000..5b93e0e --- /dev/null +++ b/src/jobs/job_example.py @@ -0,0 +1,16 @@ + + +# job.py +def run(params: dict | None = None): + print("business logic") + print(params) + + +def main(): + # CLI / subprocess 用の入口 + params = {"env": "dev"} + run(params) + + +if __name__ == "__main__": + main() diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..e93d2ad --- /dev/null +++ b/src/main.py @@ -0,0 +1,11 @@ +"""jobを実行するためのエントリポイント + +Note: Cloud Run Jobの場合はこちらから実行されます。 +""" +import jobs.job_example + +def main(): + jobs.job_example.main() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/utils/cusom_logger.py b/src/utils/cusom_logger.py new file mode 100644 index 0000000..e69de29