Add LazyIO, Fix LazyDB

Author: trisongz

.gitignore

@@ -9,4 +9,5 @@ __pycache__*
 build.sh
 *test.py
 *.egg-info*
-*.vscode
+*.vscode
+*test_ops

lazyops/__init__.py

@@ -9,7 +9,7 @@
 from .lazyclasses import lazyclass
 
 from .envs import LazyEnv, get_logger, lazywatcher, lazywatch
-from .models import LazyData, LazyTime, LazyDate, LazyFormatter, LazyTimer
+from .models import LazyData, LazyTime, LazyDate, LazyFormatter, LazyTimer, LazyObject
 from .common import lazylibs, lazy_init, run_cmd, clone_repo, File
 from .utils import find_binary_in_path, timed_cache, latest_tf_ckpt, require_module
 from .utils import build_chunks, retryable, list_to_dict
@@ -38,6 +38,7 @@
     'LazyDate',
     'LazyFormatter',
     'LazyTimer',
+    'LazyObject',
     'lazylibs',
     'lazy_init',
     'run_cmd',

lazyops/apis.py

@@ -8,7 +8,10 @@
 
 from typing import List, Dict, Any, Optional
 
-async def async_req(sess, url, method='POST', decode_json=True, with_url=False, *args, **kwargs):
+
+
+async def async_req(url, method='POST', decode_json=True, with_url=False, sess=None, headers=None, *args, **kwargs):
+    if not sess: sess = await aiohttp.ClientSession(headers=headers)
     async with sess.request(method, url=url, *args, **kwargs) as resp:
         data = await resp.json() if decode_json else await resp
         if not with_url:

lazyops/envs.py

@@ -90,7 +90,9 @@ def setup_logging(self):
         logger.propagate = self.config.get('propagate', False)
         return logger
 
-    def get_logger(self):
+    def get_logger(self, module=None):
+        if module:
+            return self.logger.getChild(module)
         return self.logger
 
     def debug(self, msg, *args, **kwargs):
@@ -122,7 +124,7 @@ def setup_new_logger(name, log_level='info', quiet_loggers=None, clear_handlers=
         'logfile_file': None,
         'logfile_log_level': "debug",
         'logfile_log_color': False,
-        'log_line_template': f"%(color_on)s[{name}] %(funcName)-5s%(color_off)s: %(message)s",
+        'log_line_template': f"%(color_on)s[{name}] %(name)-5s %(funcName)-5s%(color_off)s: %(message)s",
         'clear_handlers': clear_handlers,
         'quiet_loggers': quiet_loggers,
         'propagate': propagate
@@ -146,9 +148,11 @@ class EnvChecker:
 
 
     @classmethod
-    def get_logger(cls, name = 'LazyOps', *args, **kwargs):
+    def get_logger(cls, name = 'LazyOps', module=None, *args, **kwargs):
         if not EnvChecker.loggers.get(name):
             EnvChecker.loggers[name] = ThreadSafeHandler()
+        if module:
+            return EnvChecker.loggers[name].get(setup_new_logger, name=name, *args, **kwargs).get_logger(module=module)
         return EnvChecker.loggers[name].get(setup_new_logger, name=name, *args, **kwargs)
 
     @property

lazyops/examples/README.md

@@ -0,0 +1,2 @@
+# LazyOps Examples to Test Out
+

lazyops/lazydatabase/__init__.py

@@ -1,26 +1,29 @@
-from .schema import (
+from .backends import (
     LazyDBCacheBase,
     PklDBCache,
-    LazyDBIndex,
+)
+
+from .core import (
+    LazyDBModel, 
     LazyDBConfig,
     LazyDBBase,
     LazyDB
 )
 
-pkldb_cache = PklDBCache
-lazydb_index = LazyDBIndex
+pkldb_backend = PklDBCache
+lazydb_model = LazyDBModel
 lazydb_config = LazyDBConfig
 lazydb = LazyDB
 
 __all__ = [
     'LazyDBCacheBase',
     'PklDBCache',
-    'LazyDBIndex',
+    'LazyDBModel',
     'LazyDBConfig',
     'LazyDBBase',
     'LazyDB',
-    'pkldb_cache',
-    'lazydb_index',
+    'pkldb_backend',
+    'lazydb_model',
     'lazydb_config',
     'lazydb'
 ]

lazyops/lazydatabase/_base.py

@@ -3,21 +3,28 @@
 lazy_init('passlib[argon2]', 'passlib')
 
 import abc
+import sys
 import threading
 import time
 import base64
+import asyncio
+import fileio
+
+
+from functools import cached_property
 from dataclasses import dataclass
 from pydantic.typing import NoneType
 from pydantic import create_model, ValidationError, validator
 from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union
 from passlib.context import CryptContext
 from uuid import uuid4
+from enum import Enum
 
-import fileio
 from lazyops.lazyclasses import lazyclass
 from lazyops.lazyrpc import BaseModel, Field, sjson_dumps, sjson_loads, jsonable_encoder
-from lazyops import LazyDate, fio, get_logger, timer, tstamp, LazyEnv
+from lazyops import LazyDate, fio, get_logger, timer, tstamp, LazyEnv, LazyObject
+from lazyops import timed_cache
 
 _lazydb_default_cache_path = fio.userdir('.lazydb', mkdirs=True)
-_lazydb_logger = get_logger('LazyDB')
-_lazydb_picker = fileio.src._pickler
+baselogger = get_logger('LazyDB')
+pkler = fileio.src._pickler

lazyops/lazydatabase/backends.py

@@ -0,0 +1,125 @@
+from ._base import *
+from .models import LazyDBCacheBase
+
+logger = get_logger('LazyDB', module='backends')
+
+class PklDBCache(LazyDBCacheBase):
+    def dumps(self, data, *args, **kwargs):
+        return pkler.dumps(data, *args, **kwargs)
+    
+    def loads(self, data, *args, **kwargs):
+        return pkler.loads(data, *args, **kwargs)
+
+    def restore(self, *args, **kwargs):
+        self.lock.acquire()
+        dbdata = {}
+        with self.lock:
+            try:
+                dbdata = fio.pklload(self.cache_filepath)
+            except Exception as e:
+                logger.error(f'Failed to Restore DB {self._cachename}: {str(e)}')
+                logger.error(f'Copying DB to Backup')
+                tempfn = fio.mod_fname(self.cache_filepath, suffix='_' + tstamp())
+                fio.copy(self.cache_filepath, tempfn)
+                logger.error(f'DB saved to {tempfn}')
+        self.lock.release()
+        return dbdata
+    
+    def save(self, dbdata, *args, **kwargs):
+        _saved = False
+        self.lock.acquire()
+        with self.lock:
+            try:
+                fio.pklsave(dbdata, self.cache_filepath)
+                _saved = True
+            except Exception as e:
+                logger.error(f'Failed to Save DB {self._cachename}: {str(e)}')
+                _saved = False
+        self.lock.release()
+        return _saved
+
+class GSheetsDBCache(LazyDBCacheBase):
+    def __init__(self, sheets_url=None, cache_name='LazyCacheDB', auth_path=None, *args, **kwargs):
+        self.url = sheets_url
+        self.auth = auth_path
+        self.cache_name = cache_name
+        self.indexes = {}
+        self.setup_client()
+
+    def setup_client(self):
+        gspread = lazy_init('gspread')
+        self.gc = gspread.service_account(filename=self.auth)
+        if self.url:
+            self.sheet = self.gc.open_by_url(self.url)
+            logger.info(f'Loaded GSheetsDB from URL: {self.url}')
+        else:
+            try:
+                self.sheet = self.gc.open(self.cache_name)
+                logger.info(f'Loaded GSheetsDB from Name: {self.cache_name}')
+            
+            except Exception as e:
+                self.sheet = None
+                logger.error(f'Failed to Load GSheetsDB from Name: {self.cache_name}')
+        
+        if not self.sheet:
+            self.sheet = self.gc.create(self.cache_name)
+            logger.info(f'Created GSheetsDB by Name: {self.cache_name}')
+            self.url = self.sheet.url
+        self.refresh_index()
+    
+    def create_indexes(self, dbdata):
+        for n, i in enumerate(dbdata):
+            if i not in self.all_wks:
+                idx = dbdata[i]
+                wks = self.sheet.add_worksheet(i, index=n)
+                # Create row 0 - header
+                wks.append_row(idx.schema_props)
+
+    def dump_indexes(self, dbdata):
+        for schema, idx in dbdata.items():
+            wks = self.sheet.worksheet(schema)
+            items = []
+            for i in idx.index.values():
+                d = i.dict()
+                items.append(list(d.values()))
+            wks.insert_rows(items, row=2)
+        self.refresh_index()
+
+    def refresh_index(self):
+        self.index = self.all_wks_dict
+    
+    def get_header(self, wks):
+        return wks.row_values(1)
+
+    @property
+    def all_wks(self):
+        return list(self.sheet.worksheets())
+    
+    @property
+    def all_wks_dict(self):
+        return {n: s for n,s in enumerate(self.all_wks)}
+    
+    @property
+    def cache_file(self):
+        return None
+
+    @property
+    def cache_filepath(self):
+        return None
+    
+    @property
+    def exists(self):
+        return bool(self.url)
+    
+    # Need to figure out how to reconstruct the data
+    def dumps(self, data, *args, **kwargs):
+        pass
+    
+    def loads(self, *args, **kwargs):
+        pass
+
+    def restore(self, *args, **kwargs):
+        pass
+    
+    def save(self, db, *args, **kwargs):
+        pass