chore(ml): improved logging (#3918)

* fixed `minScore` not being set correctly

* apply to init

* don't send `enabled`

* fix eslint warning

* added logger

* added logging

* refinements

* enable access log for info level

* formatting

* merged strings

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
Mert 2023-08-30 04:22:01 -04:00 committed by GitHub
parent df26e12db6
commit 54b2779b79
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 92 additions and 11 deletions

View file

@ -1,4 +1,5 @@
import asyncio
import logging
import os
from concurrent.futures import ThreadPoolExecutor
from typing import Any
@ -11,7 +12,7 @@ from starlette.formparsers import MultiPartParser
from app.models.base import InferenceModel
from .config import settings
from .config import log, settings
from .models.cache import ModelCache
from .schemas import (
MessageResponse,
@ -20,14 +21,20 @@ from .schemas import (
)
MultiPartParser.max_file_size = 2**24 # spools to disk if payload is 16 MiB or larger
app = FastAPI()
def init_state() -> None:
app.state.model_cache = ModelCache(ttl=settings.model_ttl, revalidate=settings.model_ttl > 0)
log.info(
(
"Created in-memory cache with unloading "
f"{f'after {settings.model_ttl}s of inactivity' if settings.model_ttl > 0 else 'disabled'}."
)
)
# asyncio is a huge bottleneck for performance, so we use a thread pool to run blocking code
app.state.thread_pool = ThreadPoolExecutor(settings.request_threads)
log.info(f"Initialized request thread pool with {settings.request_threads} threads.")
@app.on_event("startup")
@ -77,4 +84,6 @@ if __name__ == "__main__":
port=settings.port,
reload=is_dev,
workers=settings.workers,
log_config=None,
access_log=log.isEnabledFor(logging.INFO),
)