syncing ballistica 1.7.50

dist/ba_data/python/baenv.py

@@ -19,6 +19,7 @@ from __future__ import annotations
 import os
 import sys
 import time
+import random
 import logging
 from pathlib import Path
 from dataclasses import dataclass
@@ -26,10 +27,12 @@ from typing import TYPE_CHECKING
 import __main__
 
 if TYPE_CHECKING:
-    from typing import Any
+    from typing import Any, Callable
 
     from efro.logging import LogHandler
 
+logger = logging.getLogger('ba.env')
+
 # IMPORTANT - It is likely (and in some cases expected) that this
 # module's code will be exec'ed multiple times. This is because it is
 # the job of this module to set up Python paths for an engine run, and
@@ -38,7 +41,7 @@ if TYPE_CHECKING:
 # /abs/path/to/ba_data/scripts/babase.py to ba_data/scripts/babase.py).
 # This can result in the next import of baenv loading us from our 'new'
 # location, which may or may not actually be the same file on disk as
-# the last load. Either way, however, multiple execs will happen in some
+# the last load. Either way, however, multiple execs can happen in some
 # form.
 #
 # To handle that situation gracefully, we need to do a few things:
@@ -53,8 +56,8 @@ if TYPE_CHECKING:
 
 # Build number and version of the ballistica binary we expect to be
 # using.
-TARGET_BALLISTICA_BUILD = 22381
-TARGET_BALLISTICA_VERSION = '1.7.41'
+TARGET_BALLISTICA_BUILD = 22535
+TARGET_BALLISTICA_VERSION = '1.7.51'
 
 
 @dataclass
@@ -67,6 +70,10 @@ class EnvConfig:
     #: Directory containing ba_data and any other platform-specific data.
     data_dir: str
 
+    #: Where cache files live (files generated by the app which can be
+    #: recreated if need be.
+    cache_dir: str
+
     #: Where the app's built-in Python stuff lives.
     app_python_dir: str | None
 
@@ -76,20 +83,19 @@ class EnvConfig:
     #: Where the app's bundled third party Python stuff lives.
     site_python_dir: str | None
 
-    #: Custom Python provided by the user (mods).
+    #: Where custom Python provided by the user (mods) lives.
     user_python_dir: str | None
 
-    #: We have a mechanism allowing app scripts to be overridden by
-    #: placing a specially named directory in a user-scripts dir. This is
-    #: true if that is enabled.
+    #: We have a mechanism allowing :attr:`app_python_dir` to be
+    #: overridden by placing a specially named directory in
+    #: :attr:`user_python_dir`. This is true if that is enabled.
     is_user_app_python_dir: bool
 
     #: Our fancy app log handler. This handles feeding logs, stdout, and
     #: stderr into the engine so they show up on in-app consoles, etc.
     log_handler: LogHandler | None
 
-    #: Initial data from the config.json file in the config dir. The
-    #: config file is parsed by
+    # Initial data from the ``config.json`` file in the config dir.
     initial_app_config: Any
 
     #: Timestamp when we first started doing stuff.
@@ -122,17 +128,20 @@ class _EnvGlobals:
 
 
 def did_paths_set_fail() -> bool:
-    """Did we try to set paths and fail?"""
+    """Did we try to set paths and fail?
+
+    :meta private:
+    """
     return _EnvGlobals.get().paths_set_failed
 
 
-def config_exists() -> bool:
+def env_config_exists() -> bool:
     """Has a config been created?"""
 
     return _EnvGlobals.get().config is not None
 
 
-def get_config() -> EnvConfig:
+def get_env_config() -> EnvConfig:
     """Return the active config, creating a default if none exists."""
     envglobals = _EnvGlobals.get()
 
@@ -143,7 +152,7 @@ def get_config() -> EnvConfig:
     # paths to run Ballistica apps should be explicitly calling
     # configure() first to get a full featured setup.
     if not envglobals.called_configure:
-        configure(setup_logging=False)
+        configure(setup_logging=False, setup_pycache_prefix=False)
 
     config = envglobals.config
     if config is None:
@@ -161,8 +170,11 @@ def configure(
     user_python_dir: str | None = None,
     app_python_dir: str | None = None,
     site_python_dir: str | None = None,
+    cache_dir: str | None = None,
     contains_python_dist: bool = False,
     setup_logging: bool = True,
+    setup_pycache_prefix: bool = False,
+    strict_threads_atexit: Callable[[Callable[[], None]], None] | None = None,
 ) -> None:
     """Set up the environment for running a Ballistica app.
 
@@ -170,6 +182,7 @@ def configure(
     creation. This must be called before any actual Ballistica modules
     are imported; the environment is locked in as soon as that happens.
     """
+    # pylint: disable=too-many-locals
 
     # Measure when we start doing this stuff. We plug this in to show
     # relative times in our log timestamp displays and also pass this to
@@ -200,6 +213,7 @@ def configure(
         site_python_dir,
         data_dir,
         config_dir,
+        cache_dir,
         standard_app_python_dir,
         is_user_app_python_dir,
     ) = _setup_paths(
@@ -208,14 +222,27 @@ def configure(
         site_python_dir,
         data_dir,
         config_dir,
+        cache_dir,
     )
 
+    # The one other thing we do before setting up logging is redirect
+    # our pyc files to our cache dir. We want to do this is calced so
+    # that as much stuff as possible (efro.logging), etc.) will get its
+    # pyc files made in our custom cache dir.
+    prev_pycache_prefix = sys.pycache_prefix
+    if setup_pycache_prefix:
+        sys.pycache_prefix = os.path.join(cache_dir, 'pyc')
+
     # Set up our log-handler and pipe Python's stdout/stderr into it.
     # Later, once the engine comes up, the handler will feed its logs
     # (including cached history) to the os-specific output location.
     # This means anything printed or logged at this point forward should
     # be visible on all platforms.
-    log_handler = _create_log_handler(launch_time) if setup_logging else None
+    log_handler = (
+        _create_log_handler(launch_time, strict_threads_atexit)
+        if setup_logging
+        else None
+    )
 
     # Load the raw app-config dict.
     app_config = _read_app_config(os.path.join(config_dir, 'config.json'))
@@ -226,13 +253,44 @@ def configure(
 
     # We want to always be run in UTF-8 mode; complain if we're not.
     if sys.flags.utf8_mode != 1:
-        logging.warning(
+        logger.warning(
             "Python's UTF-8 mode is not set. Running Ballistica without"
             ' it may lead to errors.'
         )
 
+    # We (possibly) set pycache_prefix above so that opt .pyc files are
+    # written to the cache directory that we just set up, but ideally
+    # Python should have been set to that value at startup so that
+    # modules we've imported up to this point get cached there too.
+    #
+    # In most cases we can actually do this by calcing/setting the same
+    # path we use here before spinning up Python, but in some cases
+    # that's impossible (such as our _modular_main path below where we
+    # are already in Python before we get a chance to parse args that
+    # affect cache path).
+    #
+    # So let's warn here any time we're trying to set up pycache_prefix
+    # but find that we're setting it to a different value than it was
+    # already set to. We can inform the user (or ourselves) how to line
+    # things up using PYTHONPYCACHEPREFIX or whatnot.
+    if setup_pycache_prefix and prev_pycache_prefix != sys.pycache_prefix:
+        logger.warning(
+            'Changing sys.pycache_prefix from %s to %s.'
+            ' For best performance, run with PYTHONPYCACHEPREFIX=%s.',
+            repr(prev_pycache_prefix),
+            repr(sys.pycache_prefix),
+            repr(sys.pycache_prefix),
+        )
+
     # Attempt to create dirs that we'll write stuff to.
-    _setup_dirs(config_dir, user_python_dir)
+    _setup_dirs(config_dir, user_python_dir, cache_dir)
+
+    # In debug builds, if we've not imported engine stuff yet, Kill off
+    # random cache files occasionally to help ensure that code responds
+    # correctly if/when the OS does the same thing.
+    if __debug__:
+        if '_babase' not in sys.modules:
+            _cache_ninja_rampage(cache_dir)
 
     # Get ssl working if needed so we can use https and all that.
     _setup_certs(contains_python_dist)
@@ -241,6 +299,7 @@ def configure(
     envglobals.config = EnvConfig(
         config_dir=config_dir,
         data_dir=data_dir,
+        cache_dir=cache_dir,
         user_python_dir=user_python_dir,
         app_python_dir=app_python_dir,
         standard_app_python_dir=standard_app_python_dir,
@@ -252,6 +311,21 @@ def configure(
     )
 
 
+def _cache_ninja_rampage(cache_dir: str) -> None:
+    assert os.path.isdir(cache_dir)
+    for basename, _dirnames, filenames in os.walk(cache_dir):
+        for fname in filenames:
+            # Let's kill one out of every 1000 files; should be a
+            # reasonable amount of chaos I think. Can recalibrate this
+            # as our average cache file count goes up.
+            if random.random() < 0.001:
+                fullpath = os.path.join(basename, fname)
+                logging.getLogger('ba.cache').debug(
+                    "Cache-ninja assasinated '%s'.", fullpath
+                )
+                os.unlink(fullpath)
+
+
 def _read_app_config(config_file_path: str) -> dict:
     """Read the app config."""
     import json
@@ -269,7 +343,7 @@ def _read_app_config(config_file_path: str) -> dict:
             config = {}
 
     except Exception:
-        logging.exception(
+        logger.exception(
             "Error reading config file '%s'.\n"
             "Backing up broken config to'%s.broken'.",
             config_file_path,
@@ -281,7 +355,7 @@ def _read_app_config(config_file_path: str) -> dict:
 
             shutil.copyfile(config_file_path, config_file_path + '.broken')
         except Exception:
-            logging.exception('Error copying broken config.')
+            logger.exception('Error copying broken config.')
         config = {}
 
     return config
@@ -310,7 +384,10 @@ def _calc_data_dir(data_dir: str | None) -> str:
     return data_dir
 
 
-def _create_log_handler(launch_time: float) -> LogHandler:
+def _create_log_handler(
+    launch_time: float,
+    strict_threads_atexit: Callable[[Callable[[], None]], None] | None,
+) -> LogHandler:
     from efro.logging import setup_logging, LogLevel
 
     log_handler = setup_logging(
@@ -319,7 +396,18 @@ def _create_log_handler(launch_time: float) -> LogHandler:
         log_stdout_stderr=True,
         cache_size_limit=1024 * 1024,
         launch_time=launch_time,
+        strict_threads=strict_threads_atexit is not None,
     )
+
+    # If we were given a strict_threads_atexit call, it means we should
+    # NOT use daemon threads but instead can use the atexit call to
+    # register a callback to gracefully exit our handler thread just
+    # before the interpreter shuts down. This is safer than using daemon
+    # threads, which can theoretically continue to use Python objs
+    # during and after interpreter shutdown.
+    if strict_threads_atexit is not None:
+        strict_threads_atexit(log_handler.shutdown)
+
     return log_handler
 
 
@@ -359,7 +447,7 @@ def _set_log_levels(app_config: dict) -> None:
         ).apply()
 
     except Exception:
-        logging.exception('Error setting log levels.')
+        logger.exception('Error setting log levels.')
 
 
 def _setup_certs(contains_python_dist: bool) -> None:
@@ -386,7 +474,10 @@ def _setup_paths(
     site_python_dir: str | None,
     data_dir: str | None,
     config_dir: str | None,
-) -> tuple[str | None, str | None, str | None, str, str, str, bool]:
+    cache_dir: str | None,
+) -> tuple[str | None, str | None, str | None, str, str, str, str, bool]:
+    # pylint: disable=too-many-positional-arguments
+
     # First a few paths we can ALWAYS calculate since they don't affect
     # Python imports:
 
@@ -398,6 +489,10 @@ def _setup_paths(
     if config_dir is None:
         config_dir = str(Path(Path.home(), '.ballisticakit'))
 
+    # By default, cache-dir is simply 'cache' under config-dir.
+    if cache_dir is None:
+        cache_dir = str(Path(config_dir, 'cache'))
+
     # Standard app-python-dir is simply ba_data/python under data-dir.
     standard_app_python_dir = str(Path(data_dir, 'ba_data', 'python'))
 
@@ -422,7 +517,8 @@ def _setup_paths(
         if app_python_dir is None:
             app_python_dir = standard_app_python_dir
 
-        # Likewise site-python-dir defaults to ba_data/python-site-packages.
+        # Likewise site-python-dir defaults to
+        # ba_data/python-site-packages.
         if site_python_dir is None:
             site_python_dir = str(
                 Path(data_dir, 'ba_data', 'python-site-packages')
@@ -447,7 +543,7 @@ def _setup_paths(
                 app_python_dir = str(check_dir)
                 is_user_app_python_dir = True
         except PermissionError:
-            logging.warning(
+            logger.warning(
                 "PermissionError checking user-app-python-dir path '%s'.",
                 check_dir,
             )
@@ -492,14 +588,18 @@ def _setup_paths(
         site_python_dir,
         data_dir,
         config_dir,
+        cache_dir,
         standard_app_python_dir,
         is_user_app_python_dir,
     )
 
 
-def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
+def _setup_dirs(
+    config_dir: str | None, user_python_dir: str | None, cache_dir: str
+) -> None:
     create_dirs: list[tuple[str, str | None]] = [
         ('config', config_dir),
+        ('cache', cache_dir),
         ('user_python', user_python_dir),
     ]
     for cdirname, cdir in create_dirs:
@@ -508,12 +608,12 @@ def _setup_dirs(config_dir: str | None, user_python_dir: str | None) -> None:
                 os.makedirs(cdir, exist_ok=True)
             except Exception:
                 # Not the end of the world if we can't make these dirs.
-                logging.warning(
+                logger.warning(
                     "Unable to create %s dir at '%s'.", cdirname, cdir
                 )
 
 
-def extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
+def _extract_arg(args: list[str], names: list[str], is_dir: bool) -> str | None:
     """Given a list of args and an arg name, returns a value.
 
     The arg flag and value are removed from the arg list. We also check
@@ -567,6 +667,7 @@ def _modular_main() -> None:
     # command line.
 
     try:
+
         # Take note that we're running via modular-main. The native
         # layer can key off this to know whether it should apply
         # sys.argv or not.
@@ -580,20 +681,21 @@ def _modular_main() -> None:
         args = sys.argv.copy()
 
         # NOTE: We need to keep these arg long/short arg versions synced
-        # to those in core_config.cc. That code parses these same args
-        # (even if it doesn't handle them in our case) and will complain
-        # if unrecognized args come through.
+        # to those in core_config.cc. That code will parse these same
+        # args (even if it doesn't do anything with them in this modular
+        # path) and will complain if unrecognized args come through.
 
         # Our -c arg basically mirrors Python's -c arg. If we get that,
         # simply exec it and return; no engine stuff.
-        command = extract_arg(args, ['--command', '-c'], is_dir=False)
+        command = _extract_arg(args, ['--command', '-c'], is_dir=False)
         if command is not None:
             exec(command)  # pylint: disable=exec-used
             return
 
-        config_dir = extract_arg(args, ['--config-dir', '-C'], is_dir=True)
-        data_dir = extract_arg(args, ['--data-dir', '-d'], is_dir=True)
-        mods_dir = extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
+        config_dir = _extract_arg(args, ['--config-dir', '-C'], is_dir=True)
+        data_dir = _extract_arg(args, ['--data-dir', '-d'], is_dir=True)
+        mods_dir = _extract_arg(args, ['--mods-dir', '-m'], is_dir=True)
+        cache_dir = _extract_arg(args, ['--cache-dir', '-a'], is_dir=True)
 
         # We run configure() BEFORE importing babase. (part of its job
         # is to wrangle paths which can affect where babase and
@@ -602,6 +704,7 @@ def _modular_main() -> None:
             config_dir=config_dir,
             data_dir=data_dir,
             user_python_dir=mods_dir,
+            cache_dir=cache_dir,
         )
 
         import babase