基本libをUP

.gitignore

@@ -1,3 +1,6 @@
+data/
+!.gitkeep
+
 # ---> Python
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -15,7 +18,7 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
+# lib/
 lib64/
 parts/
 sdist/

README.md

@@ -1,3 +1,12 @@
 # colect-data-pipe-line
 
-データ収集と蓄積までの検討及び調査
+データ収集と蓄積までの検討及び調査
+
+## Develop
+
+```sh
+python -m venv venv
+# windows
+venv\Scripts\activate
+pip install -r requirements.txt
+```

requirements.txt

@@ -0,0 +1,20 @@
+duckdb==1.3.2
+numpy==2.3.3
+pandas==2.3.2
+
+python-dotenv
+
+# RSS
+feedparser==6.0.12
+# model
+pydantic==2.11.9
+
+# deepl
+deepl
+
+# transelate argo
+#argostranslate
+
+# transformers
+# sentencepiece
+# torch

src/lib/custom_logger.py

@@ -0,0 +1,56 @@
+import logging
+import functools
+from .singleton import Singleton
+
+class CustomLogger(Singleton):
+    """
+    Singleton logger class that initializes a logger with a specified name and log file.
+    It provides a method to log entry and exit of functions.
+    """
+
+    def __init__(self, name='main', log_file=None, level=logging.INFO):
+        if hasattr(self, '_initialized') and self._initialized:
+            return  # すでに初期化済みなら何もしない
+            # self.logger.setLevel(level)
+
+        self.logger = logging.getLogger(name)
+        self.logger.setLevel(level)
+        self.logger.propagate = False
+
+        formatter = logging.Formatter(
+            '%(asctime)s %(levelname)s [%(filename)s:%(lineno)3d]: %(message)s'
+        )
+
+        # Console handler
+        ch = logging.StreamHandler()
+        ch.setFormatter(formatter)
+        self.logger.addHandler(ch)
+
+        # File handler
+        if log_file:
+            fh = logging.FileHandler(log_file, encoding='utf-8')
+            fh.setFormatter(formatter)
+            self.logger.addHandler(fh)
+
+        self._initialized = True
+
+
+    def get_logger(self):
+        return self.logger
+
+    def log_entry_exit(self, func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            self.logger.info(f"Enter: {func.__qualname__}")
+            result = func(*args, **kwargs)
+            self.logger.info(f"Exit: {func.__qualname__}")
+            return result
+        return wrapper
+
+
+
+
+
+def get_logger(name='main', log_file=None, level=logging.INFO):
+    custom_logger = CustomLogger(name, log_file, level)
+    return custom_logger.get_logger()

src/lib/singleton.py

@@ -0,0 +1,20 @@
+"""Singleton pattern implementation in Python.
+This implementation is thread-safe and ensures that only one instance of the class is created.
+
+Singleton が提供するのは「同じインスタンスを返す仕組み」
+* __init__() は毎回呼ばれる(多くの人が意図しない動作)
+* __init__の2回目は_initialized というフラグは 使う側で管理する必要がある。
+"""
+
+import threading
+
+class Singleton(object):
+    _instances = {}
+    _lock = threading.Lock()
+
+    def __new__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            with cls._lock:
+                if cls not in cls._instances:  # ダブルチェック
+                    cls._instances[cls] = super(Singleton, cls).__new__(cls)
+        return cls._instances[cls]