Initial commit
This commit is contained in:
commit
7825108009
|
|
@ -0,0 +1,19 @@
|
||||||
|
Copyright (c) 2024 Audrey Dutcher
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
{ pkgs }:
|
||||||
|
let
|
||||||
|
lib = pkgs.lib;
|
||||||
|
types = {
|
||||||
|
Module = "Module";
|
||||||
|
Class = "Class";
|
||||||
|
DataType = "DataType";
|
||||||
|
Repo = "Repo";
|
||||||
|
RepoAdapter = "RepoAdapter";
|
||||||
|
Function = "Function";
|
||||||
|
Binding = "Binding";
|
||||||
|
FunctionType = "FunctionType";
|
||||||
|
};
|
||||||
|
addType = arg: ty: arg // { __artinixType = ty; };
|
||||||
|
needsName = arg: arg // { __artinixName = null; };
|
||||||
|
addName = name: arg: if arg ? __artinixName && arg.__artinixName == null
|
||||||
|
then arg // { __artinixName = name; }
|
||||||
|
else arg;
|
||||||
|
addNames = lib.mapAttrs addName;
|
||||||
|
moduleExtension = final: prev: (addType (addNames prev) types.Module);
|
||||||
|
classExtension = final: prev: (addType (needsName prev) types.Class);
|
||||||
|
repoExtension = class: repoType: final: prev: (addType (needsName (repoType class prev)) types.Repo);
|
||||||
|
dataTypeExtension = final: prev: (addType (needsName prev) types.DataType) // { __functor = self: class: attrs: attrs // { __artinixRepoType = self; __artinixClass = class; }; };
|
||||||
|
adapterExtension = final: prev: (addType (needsName prev) types.RepoAdapter);
|
||||||
|
functionExtension = functy: final: prev: (addType (needsName (functy prev)) types.Function);
|
||||||
|
bindingExtension = class: final: prev: (addType (needsName prev) types.Binding) // { __artinixClass = class; };
|
||||||
|
functionTypeExtension = typeCons: final: prev: (addType prev types.FunctionType) // { __functor = self: args: (typeCons args) // { __artinixFunctionType = self; }; };
|
||||||
|
constructor = extension: mod: (lib.makeExtensible mod).extend extension;
|
||||||
|
in
|
||||||
|
rec {
|
||||||
|
mkModule = constructor moduleExtension;
|
||||||
|
mkClass = constructor classExtension;
|
||||||
|
mkDataType = constructor dataTypeExtension;
|
||||||
|
mkRepo = cls: repoty: constructor (repoExtension cls repoty);
|
||||||
|
mkAdapter = constructor adapterExtension;
|
||||||
|
mkFuncType = typeCons: constructor (functionTypeExtension typeCons);
|
||||||
|
mkFunc = functy: constructor (functionExtension functy);
|
||||||
|
mkBinding = cls: constructor (bindingExtension cls);
|
||||||
|
mkView = cls: functy: args: mkBinding cls { func = mkFunc functy (builtins.removeAttrs args ["inputs" "outputs"]); inherit (args) inputs outputs; };
|
||||||
|
mkRuntime = ({nativeFunctionTypes, nativeDataFormats, linker}: );
|
||||||
|
mkProgram = ({runtime, bindings, storageProviders}: );
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
{
|
||||||
|
pkgs,
|
||||||
|
lib,
|
||||||
|
stdlib,
|
||||||
|
jqlib,
|
||||||
|
pythonlib, # TODO python env...
|
||||||
|
processlib,
|
||||||
|
}:
|
||||||
|
with lib;
|
||||||
|
mkModule (self: {
|
||||||
|
# A class is basically an identifier namespace.
|
||||||
|
class1 = mkClass {};
|
||||||
|
# Another class of entities. May have overlapping identifiers with class1 but there is no correlation
|
||||||
|
class2 = mkClass {};
|
||||||
|
|
||||||
|
# A repository is an attribute of an entity of a given class. It is the core data storage abstraction.
|
||||||
|
repo1 = mkRepo self.class1 stdlib.dtypes.blob {};
|
||||||
|
repo2 = mkRepo self.class1 stdlib.dtypes.json {
|
||||||
|
schema = with stdlib.dtypes.json.schema; dictKeysComplete { related = int; subkey = dictOf str (listOf str); };
|
||||||
|
};
|
||||||
|
# A repository can be declared on each class
|
||||||
|
repo3 = mkRepo self.class2 stdlib.dtypes.json {};
|
||||||
|
repo4 = mkRepo self.class2 stdlib.dtypes.blob {};
|
||||||
|
|
||||||
|
# A view is a derived repository. There are many ways to describe a transformation between inputs and outputs.
|
||||||
|
# Here we use jq to query over an input repository. We also cast it to a foreign key, which is the id of an entity of a given class.
|
||||||
|
relation = mkView self.class1 jqlib.expr {
|
||||||
|
inputs.input.repo = self.repo2;
|
||||||
|
query = "$input.related";
|
||||||
|
cast = stdlib.dtypes.foreignKeyOf self.class2;
|
||||||
|
};
|
||||||
|
# Here, we use the previous foreign key repository to create a new repository on class1 which contains data from repo3 (declared on class2)
|
||||||
|
# by specifying that the this repository should be accessed through a foreign entity.
|
||||||
|
repoRelated = mkRelated self.class1 self.relation self.repo3 {};
|
||||||
|
|
||||||
|
# The declaration of a view is a shorthand for declaring a function and binding it to some input and output repositories.
|
||||||
|
# Here we do it in long form
|
||||||
|
func1 = mkFunc pythonlib.func {
|
||||||
|
inputs.one.dtype = stdlib.dtypes.json;
|
||||||
|
inputs.two.dtype = stdlib.dtypes.blob;
|
||||||
|
outputs.return.dtype = stdlib.dtypes.json;
|
||||||
|
|
||||||
|
module = pkgs.writeText "func1.py" ''
|
||||||
|
def func1(one, two):
|
||||||
|
return {
|
||||||
|
"one": one["yay"],
|
||||||
|
"two": two.read(),
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
function = "func1";
|
||||||
|
};
|
||||||
|
bind1 = mkBinding self.class1 {
|
||||||
|
func = self.func1;
|
||||||
|
inputs.one = self.repo2;
|
||||||
|
inputs.two = self.repo1;
|
||||||
|
outputs.return = self.repoRelated;
|
||||||
|
};
|
||||||
|
|
||||||
|
streamFunc = mkFunc processlib.processFunc {
|
||||||
|
inputs.stdin = {
|
||||||
|
dtype = stdlib.dtypes.json;
|
||||||
|
format = processlib.formats.streamOf processlib.formats.yaml;
|
||||||
|
};
|
||||||
|
outputs.out = {
|
||||||
|
type = stdlib.dtypes.seqOf stdlib.dtypes.blob;
|
||||||
|
format = processlib.formats.filepathOf (processlib.formats.watchdirOf processlib.formats.file);
|
||||||
|
};
|
||||||
|
|
||||||
|
executable = pkgs.writeShellScript "streamFunc.sh" ''
|
||||||
|
id=0
|
||||||
|
grep whatever | while read -r line; do
|
||||||
|
md5sum >$out/$id <<<$line
|
||||||
|
id=$((id + 1))
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
tupleFunc = mkFunc processlib.processFunc {
|
||||||
|
inputs.stdin = {
|
||||||
|
dtype = stdlib.dtypes.json;
|
||||||
|
format = processlib.formats.streamOf processlib.formats.yaml;
|
||||||
|
};
|
||||||
|
outputs.out = {
|
||||||
|
type = stdlib.dtypes.seqOf (stdlib.dtypes.tupleOf { data = stdlib.dtypes.blob; metadata = stdlib.dtypes.json; });
|
||||||
|
format = processlib.formats.filepathOf (processlib.formats.watchdirOf (processlib.formats.tupleDirOf {
|
||||||
|
data = processlib.formats.file;
|
||||||
|
metadata = processlib.formats.yaml;
|
||||||
|
}));
|
||||||
|
};
|
||||||
|
|
||||||
|
executable = pkgs.writeShellScript "tupleFunc.sh" ''
|
||||||
|
id=0
|
||||||
|
grep whatever | while read -r line; do
|
||||||
|
mkdir -p $out/$id
|
||||||
|
md5sum >$out/$id/data <<<$line
|
||||||
|
cat >$out/$id/metadata <<EOF
|
||||||
|
parent: $localKey
|
||||||
|
filesize: $(wc -c $out/$id/data)
|
||||||
|
EOF
|
||||||
|
id=$((id + 1))
|
||||||
|
done
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
bind2 = mkBinding self.class1 {
|
||||||
|
func = self.tupleFunc;
|
||||||
|
inputs.stdin = self.repo2;
|
||||||
|
output.out = stdlib.repoAdapters.seqToSpawn (stdlib.repoAdapters.tupleToRepos {
|
||||||
|
data = self.repo4;
|
||||||
|
metadata = self.repo3;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
parentKey = mkView self.class2 jqlib.expr {
|
||||||
|
inputs.input.repo = self.repo3;
|
||||||
|
query = "$input.parent";
|
||||||
|
cast = stdlib.dtypes.foreignKeyOf self.class1;
|
||||||
|
};
|
||||||
|
})
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
mkProgram,
|
||||||
|
callModule,
|
||||||
|
myModule ? callModule ./module.nix,
|
||||||
|
}:
|
||||||
|
mkProgram self: {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
{
|
||||||
|
mkModule,
|
||||||
|
mkClass,
|
||||||
|
mkRepo,
|
||||||
|
mkView,
|
||||||
|
stdlib,
|
||||||
|
jqlib,
|
||||||
|
}:
|
||||||
|
mkModule (self: {
|
||||||
|
artifact = mkClass {
|
||||||
|
description = "A file that will be parsed and interrogated";
|
||||||
|
};
|
||||||
|
|
||||||
|
artifactBody = mkRepo self.artifact stdlib.repos.blob {};
|
||||||
|
artifactMeta = mkRepo self.artifact stdlib.repos.json {
|
||||||
|
schema = with stdlib.repos.json.schema; dictKeysIncomplete {
|
||||||
|
filename = str;
|
||||||
|
sha256 = str;
|
||||||
|
source = any;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
executable = mkClass {
|
||||||
|
description = "A file that can be parsed as a program";
|
||||||
|
superclass = self.artifact;
|
||||||
|
};
|
||||||
|
|
||||||
|
executableMeta = mkRepo self.executable stdlib.repos.json {
|
||||||
|
schema = with stdlib.repos.json.schema; dictKeysIncomplete {
|
||||||
|
format = str;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
})
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
# Artinix - Python Runtime
|
||||||
|
|
||||||
|
This is a runtime for efficiently integrating python functions and packages into an Artinix pipeline.
|
||||||
|
|
@ -0,0 +1,195 @@
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
pkgs,
|
||||||
|
}:
|
||||||
|
with lib;
|
||||||
|
let
|
||||||
|
indent =
|
||||||
|
indent: text:
|
||||||
|
pkgs.lib.strings.concatMapLines (line: indent + line + "\n") (
|
||||||
|
if builtins.isList text then text else pkgs.lib.strings.splitString "\n" text
|
||||||
|
);
|
||||||
|
pyBool = val: if val then "True" else "False";
|
||||||
|
pyStr = val: "\"${builtins.replaceStrings ["\\" "\"" "\n"] ["\\\\" "\\\"" "\\n"] val}\"";
|
||||||
|
in
|
||||||
|
mkModule (self: {
|
||||||
|
pythonRuntime = mkRuntime (self: {
|
||||||
|
nativeFunctionTypes = [ self.func ];
|
||||||
|
nativeDataFormats = with self.formats; [
|
||||||
|
jsonObject
|
||||||
|
blobStream
|
||||||
|
blobString
|
||||||
|
];
|
||||||
|
python = pkgs.python3;
|
||||||
|
builder =
|
||||||
|
bindings:
|
||||||
|
let
|
||||||
|
initialState = {
|
||||||
|
repositories = [ ];
|
||||||
|
bindings = [ ];
|
||||||
|
functions = [ ];
|
||||||
|
classes = [ ];
|
||||||
|
};
|
||||||
|
stateAppend =
|
||||||
|
state: kind: obj:
|
||||||
|
state // { ${kind} = state.${kind} ++ [ obj ]; };
|
||||||
|
findOrAssign =
|
||||||
|
lst: obj:
|
||||||
|
let
|
||||||
|
newIndex = builtins.length lst;
|
||||||
|
index = pkgs.lib.lists.findFirstIndex (obj2: obj == obj2) newIndex lst;
|
||||||
|
new = index == newIndex;
|
||||||
|
newLst = lst ++ (if new then [ obj ] else [ ]);
|
||||||
|
in
|
||||||
|
{
|
||||||
|
inherit index new;
|
||||||
|
lst = newLst;
|
||||||
|
};
|
||||||
|
|
||||||
|
typeToAttr = {
|
||||||
|
Function = "functions";
|
||||||
|
Repo = "repositories";
|
||||||
|
Class = "classes";
|
||||||
|
Binding = "bindings";
|
||||||
|
};
|
||||||
|
|
||||||
|
visit = visitors: obj: visitors.${typeToAttr.${obj.__artinixType}} obj;
|
||||||
|
childrenOf = obj: visit childVisitors obj;
|
||||||
|
childVisitors = {
|
||||||
|
repositories =
|
||||||
|
repository:
|
||||||
|
initialState
|
||||||
|
// {
|
||||||
|
classes = [ repository.class ];
|
||||||
|
};
|
||||||
|
classes = class: initialState;
|
||||||
|
functions = func: initialState;
|
||||||
|
bindings =
|
||||||
|
binding:
|
||||||
|
initialState
|
||||||
|
// {
|
||||||
|
classes = [ binding.class ];
|
||||||
|
repositories =
|
||||||
|
(mapAttrsToList (_: r: r) bindings.inputs) ++ (mapAttrsToList (_: r: r) bindings.outputs);
|
||||||
|
functions = [ binding.function ];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
explode' = state: obj: let
|
||||||
|
lst = state.${typeToAttr.${obj.__artinixType}};
|
||||||
|
found = findOrAssign lst obj;
|
||||||
|
in (if found.new then (let
|
||||||
|
state1 = stateAppend state typeToAttr.${obj.__artinixType} obj;
|
||||||
|
children = childrenOf obj;
|
||||||
|
state3 = foldlAttrs (state2: _: childrenList: builtins.foldl' explode' state2 childrenList) state1 children;
|
||||||
|
in state3) else state);
|
||||||
|
explodeList = builtins.foldl' explode' initialState;
|
||||||
|
|
||||||
|
indexOf = item: lst: findFirstIndex (obj: obj == item) (throw "Option was not found in the options list") lst;
|
||||||
|
reloop = exploded: let
|
||||||
|
numberVisitors = {
|
||||||
|
repositories = repo: repo // { class = indexOf repo.class exploded.classes; };
|
||||||
|
classes = class: class;
|
||||||
|
functions = func: func;
|
||||||
|
bindings = binding: binding // {
|
||||||
|
class = indexOf binding.class exploded.classes;
|
||||||
|
function = indexOf binding.function exploded.functions;
|
||||||
|
inputs = builtins.mapAttrs (k: v: indexOf v exploded.repositories) binding.inputs;
|
||||||
|
outputs = builtins.mapAttrs (k: v: indexOf v exploded.repositories) binding.outputs;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
numbered = builtins.mapAttrs (k: v: builtins.map numberVisitors.${k} v) exploded;
|
||||||
|
result = pkgs.lib.converge (final: let
|
||||||
|
reloopVisitors = {
|
||||||
|
repositories = repo: repo // { class = builtins.elemAt final.classes repo.class; };
|
||||||
|
classes = class: class;
|
||||||
|
functions = func: func;
|
||||||
|
bindings = binding: binding // {
|
||||||
|
class = builtins.elemAt final.classes binding.class;
|
||||||
|
function = builtins.elemAt final.functions binding.function;
|
||||||
|
inputs = builtins.mapAttrs (k: v: builtins.elemAt final.repositories v) binding.inputs;
|
||||||
|
outputs = builtins.mapAttrs (k: v: builtins.elemAt final.repositories v) binding.outputs;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
in
|
||||||
|
builtins.mapAttrs (k: v: pkgs.lib.imap0 (i: v': (reloopVisitors.${k} v') // { __index = i; }) v) final
|
||||||
|
) numbered;
|
||||||
|
in result;
|
||||||
|
|
||||||
|
#index' = exploded: obj: let
|
||||||
|
# options = exploded.${typeToAttr.${obj.__artinixType}};
|
||||||
|
# index = indexOf obj options;
|
||||||
|
# relooped = reloop exploded;
|
||||||
|
# options' = relooped.${typeToAttr.${obj.__artinixType}};
|
||||||
|
#in builtins.elemAt options' index;
|
||||||
|
#indexList = lst: let exploded = explodeList lst; in builtins.map (index' exploded) lst;
|
||||||
|
#index = obj: index' (explode obj) obj;
|
||||||
|
#bindings = indexList bindings;
|
||||||
|
|
||||||
|
finalState = reloop (explodeList bindings);
|
||||||
|
|
||||||
|
inherit (pkgs.lib.attrsets) mapAttrsToList foldlAttrs;
|
||||||
|
inherit (pkgs.lib.lists) findFirstIndex;
|
||||||
|
|
||||||
|
allPyDeps = pkgs.lib.lists.concatMap (func: func.pythonDeps) attrs.nativeFunctions;
|
||||||
|
pyEnv = self.python.withPackages (
|
||||||
|
pythonPackages:
|
||||||
|
builtins.map (dep: if builtins.isString dep then pythonPackages.${dep} else dep) allPyDeps
|
||||||
|
);
|
||||||
|
in
|
||||||
|
pkgs.substituteAll {
|
||||||
|
name = "driver.py";
|
||||||
|
src = ./src/driver.py.template;
|
||||||
|
interpreter = pyEnv.mainProgram;
|
||||||
|
allowDefault = pyBool TODO;
|
||||||
|
allowManual = pyBool TODO;
|
||||||
|
entityClassSetup = indent "" (
|
||||||
|
pkgs.lib.flip builtins.map finalState.bindings (
|
||||||
|
binding: "entity_classes.append(ArtinixEntityClass(${pyStr binding.name}))"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
repositorySetup = indent "" (
|
||||||
|
pkgs.lib.flip builtins.map finalState.repositories (
|
||||||
|
repo: "repositories.append(ArtinixRepository(entity_classes[${repo.class.__index}], ${pyStr repo.name}, ${pystr "dtype?"}))"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
functionSetup = indent "" (
|
||||||
|
pkgs.lib.flip builtins.map finalState.functions (
|
||||||
|
func: "functions.append(ArtinixPythonFunction(${pyStr func.module_path}, ${pyStr func.func_name}))"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
datastoreSetup = "";
|
||||||
|
};
|
||||||
|
});
|
||||||
|
# TODO integrate the typechecker right here I think?
|
||||||
|
func = mkFuncType (
|
||||||
|
{ module, function }:
|
||||||
|
{
|
||||||
|
linker = self.pythonLinker;
|
||||||
|
inherit module function;
|
||||||
|
}
|
||||||
|
);
|
||||||
|
pythonTypeChecker = (
|
||||||
|
{
|
||||||
|
module,
|
||||||
|
function,
|
||||||
|
inputs,
|
||||||
|
outputs,
|
||||||
|
}:
|
||||||
|
pkgs.runCommand "typecheck-python-${builtins.baseNameOf module}-${function}" { } ''
|
||||||
|
# tee hee
|
||||||
|
touch $out
|
||||||
|
''
|
||||||
|
);
|
||||||
|
formats = mkModule (self': {
|
||||||
|
jsonObject = mkFormat { pythonConstructor = "ArtinixJsonObjectFormat()"; };
|
||||||
|
blobStream = mkFormat { pythonConstructor = "ArtinixBlobStreamFormat()"; };
|
||||||
|
blobString = mkFormat { pythonConstructor = "ArtinixBlobStringFormat()"; };
|
||||||
|
pythonObject = ty: mkFormat { inherit ty; pythonConstructor = "ArtinixPythonObjectFormat(${pyStr ty})"; };
|
||||||
|
generatorOf = subfmt: mkFormat { inherit subfmt; pythonConstructor = "ArtinixGeneratorOfFormat(${subfmt.pythonConstructor})"; };
|
||||||
|
});
|
||||||
|
dtypes = mkModule (self': {
|
||||||
|
pythonClass = name: mkRepoType { inherit name; };
|
||||||
|
});
|
||||||
|
})
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "artinix-python"
|
||||||
|
version = "0.0.1"
|
||||||
|
authors = [
|
||||||
|
{ name="Audrey Dutcher", email="audrey@rhelmot.io" },
|
||||||
|
]
|
||||||
|
description = "Python runtime for Artinix"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.8"
|
||||||
|
classifiers = [
|
||||||
|
"Programming Language :: Python :: 3",
|
||||||
|
"License :: OSI Approved :: MIT License",
|
||||||
|
"Operating System :: OS Independent",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Homepage = "https://github.com/rhelmot/artinix"
|
||||||
|
Issues = "https://github.com/rhelmot/artinix/issues"
|
||||||
|
|
@ -0,0 +1,151 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from collections.abc import Iterable
|
||||||
|
import importlib.util
|
||||||
|
from types import ModuleType
|
||||||
|
from typing import Any, Protocol
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import random
|
||||||
|
import string
|
||||||
|
|
||||||
|
PYTHON_MODULE_CACHE: dict[str, ModuleType] = {}
|
||||||
|
|
||||||
|
class ArtinixStorage(Protocol):
|
||||||
|
def members(self) -> Iterable[str]:
|
||||||
|
...
|
||||||
|
|
||||||
|
def has(self, ident: str) -> bool:
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
|
class ArtinixFormat(Protocol):
|
||||||
|
def get(self, storage: ArtinixStorage, ident: str) -> Any:
|
||||||
|
...
|
||||||
|
|
||||||
|
def reader(self, storage: ArtinixStorage, ident: str) -> Any | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def put(self, storage: ArtinixStorage, ident: str, data: Any) -> None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def writer(self, storage: ArtinixStorage, ident: str) -> Any | None:
|
||||||
|
...
|
||||||
|
|
||||||
|
def filter(self, storage: type[ArtinixStorage]) -> bool:
|
||||||
|
...
|
||||||
|
|
||||||
|
class ArtinixFunction(Protocol):
|
||||||
|
def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
...
|
||||||
|
|
||||||
|
input_formats: dict[str, ArtinixFormat]
|
||||||
|
output_formats: dict[str, ArtinixFormat]
|
||||||
|
|
||||||
|
class ArtinixPythonFunction:
|
||||||
|
def __init__(self, module_path: str, func_name: str, input_formats: dict[str, ArtinixFormat], output_formats: dict[str, ArtinixFormat]):
|
||||||
|
self.module_path = module_path
|
||||||
|
self.func_name = func_name
|
||||||
|
self.input_formats = input_formats
|
||||||
|
self.output_formats = output_formats
|
||||||
|
|
||||||
|
module = PYTHON_MODULE_CACHE.get(self.module_path, None)
|
||||||
|
if module is None:
|
||||||
|
spec = importlib.util.spec_from_file_location(module_path, module_path)
|
||||||
|
if spec is None:
|
||||||
|
raise FileNotFoundError(f"{module_path} not found")
|
||||||
|
assert spec.loader is not None, "What does it even mean when spec.loader is None?"
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
PYTHON_MODULE_CACHE[module_path] = module
|
||||||
|
self.module = module
|
||||||
|
self.function = getattr(module, self.func_name)
|
||||||
|
|
||||||
|
def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
return {"return": self.function(**inputs)}
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ArtinixEntityClass:
|
||||||
|
name: str
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ArtinixEntity:
|
||||||
|
ident: str
|
||||||
|
klass: ArtinixEntityClass
|
||||||
|
|
||||||
|
|
||||||
|
class ArtinixRepository:
|
||||||
|
def __init__(self, klass: ArtinixEntityClass, name: str, dtype: str):
|
||||||
|
self.klass = klass
|
||||||
|
self.name = name
|
||||||
|
self.dtype = dtype
|
||||||
|
|
||||||
|
class ArtinixBinding:
|
||||||
|
def __init__(self, klass: ArtinixEntityClass, function: ArtinixFunction, inputs: dict[str, ArtinixRepository], outputs: dict[str, ArtinixRepository]):
|
||||||
|
self.klass = klass
|
||||||
|
self.function = function
|
||||||
|
self.inputs = inputs
|
||||||
|
self.outputs = outputs
|
||||||
|
|
||||||
|
assert inputs, "Binding must have inputs so we know when to trigger it"
|
||||||
|
|
||||||
|
class ArtinixPythonRuntime:
|
||||||
|
def __init__(self,
|
||||||
|
*,
|
||||||
|
argv0: str,
|
||||||
|
default_setup: bool,
|
||||||
|
manual_setup: bool,
|
||||||
|
bindings: list[ArtinixBinding],
|
||||||
|
datastore: dict[ArtinixRepository, ArtinixStorage],
|
||||||
|
):
|
||||||
|
|
||||||
|
self.bindings = bindings
|
||||||
|
self.datastore = datastore
|
||||||
|
|
||||||
|
self.parser = argparse.ArgumentParser(argv0)
|
||||||
|
subparsers = self.parser.add_subparsers()
|
||||||
|
|
||||||
|
if manual_setup:
|
||||||
|
args_setup = subparsers.add_parser("setup")
|
||||||
|
args_setup.set_defaults(func=self.func_setup)
|
||||||
|
|
||||||
|
if default_setup:
|
||||||
|
args_run = subparsers.add_parser("run")
|
||||||
|
args_run.set_defaults(func=self.func_run)
|
||||||
|
|
||||||
|
def spawn_entity(self, klass: ArtinixEntityClass) -> ArtinixEntity:
|
||||||
|
return ArtinixEntity(ident = ''.join(random.choice(string.ascii_lowercase) for _ in range(8)), klass=klass)
|
||||||
|
|
||||||
|
def binding_ready(self, binding: ArtinixBinding) -> list[ArtinixEntity]:
|
||||||
|
ready = None
|
||||||
|
for input_repo in binding.inputs.values():
|
||||||
|
storage = self.datastore[input_repo]
|
||||||
|
if ready is None:
|
||||||
|
ready = set(storage.members())
|
||||||
|
else:
|
||||||
|
ready.intersection_update(storage.members())
|
||||||
|
|
||||||
|
assert ready is not None
|
||||||
|
for output_repo in binding.outputs.values():
|
||||||
|
storage = self.datastore[output_repo]
|
||||||
|
ready.difference_update(storage.members())
|
||||||
|
|
||||||
|
return [ArtinixEntity(ident=ident, klass=binding.klass) for ident in ready]
|
||||||
|
|
||||||
|
def binding_exec(self, binding: ArtinixBinding, entity: ArtinixEntity):
|
||||||
|
inputs = {input_name: binding.function.input_formats[input_name].get(self.datastore[input_repo], entity.ident) for input_name, input_repo in binding.inputs.items()}
|
||||||
|
outputs = binding.function(inputs)
|
||||||
|
for output_name, output_data in outputs.items():
|
||||||
|
binding.function.output_formats[output_name].put(self.datastore[binding.outputs[output_name]], entity.ident, output_data)
|
||||||
|
|
||||||
|
def main(self, args: list[str]):
|
||||||
|
parsed = self.parser.parse_args(args)
|
||||||
|
func_args = vars(parsed)
|
||||||
|
func = func_args.pop("func")
|
||||||
|
return func(**func_args)
|
||||||
|
|
||||||
|
def func_setup(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def func_run(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
from typing import TYPE_CHECKING, Any
|
||||||
|
from io import BytesIO
|
||||||
|
import json
|
||||||
|
|
||||||
|
from .storage import *
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from . import ArtinixStorage
|
||||||
|
|
||||||
|
__all__ = ('ArtinixJsonObjectFormat', 'ArtinixBlobStreamFormat', 'ArtinixBlobStringFormat')
|
||||||
|
|
||||||
|
|
||||||
|
class ArtinixJsonObjectFormat:
|
||||||
|
def filter(self, storage: type[ArtinixStorage]) -> bool:
|
||||||
|
return issubclass(storage, (ArtinixInMemoryStorage, ArtinixFileStorage))
|
||||||
|
|
||||||
|
def get(self, storage: ArtinixStorage, ident: str):
|
||||||
|
if isinstance(storage, ArtinixInMemoryStorage):
|
||||||
|
data = storage.data[ident]
|
||||||
|
assert isinstance(data, (int, float, bool, str, list, dict))
|
||||||
|
return data
|
||||||
|
if isinstance(storage, ArtinixFileStorage):
|
||||||
|
with open(storage.basedir / ident, 'r') as fp:
|
||||||
|
return json.load(fp)
|
||||||
|
raise NotImplementedError("Forgot a case")
|
||||||
|
|
||||||
|
def put(self, storage: ArtinixStorage, ident: str, data: Any) -> None:
|
||||||
|
assert isinstance(data, (int, float, bool, str, list, dict))
|
||||||
|
if isinstance(storage, ArtinixInMemoryStorage):
|
||||||
|
storage.data[ident] = data
|
||||||
|
if isinstance(storage, ArtinixFileStorage):
|
||||||
|
with open(storage.basedir / ident, 'w') as fp:
|
||||||
|
return json.dump(data, fp)
|
||||||
|
raise NotImplementedError("Forgot a case")
|
||||||
|
|
||||||
|
def writer(self, storage, ident):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def reader(self, storage, ident):
|
||||||
|
return None
|
||||||
|
|
||||||
|
class ArtinixBlobStreamFormat:
|
||||||
|
def filter(self, storage: type[ArtinixStorage]):
|
||||||
|
return issubclass(storage, (ArtinixInMemoryStorage, ArtinixFileStorage))
|
||||||
|
|
||||||
|
def get(self, storage, ident):
|
||||||
|
raise TypeError("Please use reader")
|
||||||
|
|
||||||
|
def put(self, storage, ident):
|
||||||
|
raise TypeError("Please use writer")
|
||||||
|
|
||||||
|
def reader(self, storage: ArtinixStorage, ident: str):
|
||||||
|
if isinstance(storage, ArtinixInMemoryStorage):
|
||||||
|
data = storage.data[ident]
|
||||||
|
assert isinstance(data, BytesIO)
|
||||||
|
return data
|
||||||
|
if isinstance(storage, ArtinixFileStorage):
|
||||||
|
return open(storage.basedir / ident, 'rb')
|
||||||
|
raise NotImplementedError("Forgot a case")
|
||||||
|
|
||||||
|
def writer(self, storage: ArtinixStorage, ident: str):
|
||||||
|
if isinstance(storage, ArtinixInMemoryStorage):
|
||||||
|
data = storage.data[ident]
|
||||||
|
assert isinstance(data, BytesIO)
|
||||||
|
return data
|
||||||
|
if isinstance(storage, ArtinixFileStorage):
|
||||||
|
return open(storage.basedir / ident, 'wb')
|
||||||
|
raise NotImplementedError("Forgot a case")
|
||||||
|
|
||||||
|
class ArtinixBlobStringFormat:
|
||||||
|
def filter(self, storage: type[ArtinixStorage]) -> bool:
|
||||||
|
return issubclass(storage, (ArtinixInMemoryStorage, ArtinixFileStorage))
|
||||||
|
|
||||||
|
def get(self, storage: ArtinixStorage, ident: str):
|
||||||
|
if isinstance(storage, ArtinixInMemoryStorage):
|
||||||
|
data = storage.data[ident]
|
||||||
|
assert isinstance(data, BytesIO)
|
||||||
|
return data.getvalue()
|
||||||
|
if isinstance(storage, ArtinixFileStorage):
|
||||||
|
with open(storage.basedir / ident, 'rb') as fp:
|
||||||
|
return fp.read()
|
||||||
|
raise NotImplementedError("Forgot a case")
|
||||||
|
|
||||||
|
def put(self, storage: ArtinixStorage, ident: str, data: Any) -> None:
|
||||||
|
assert isinstance(data, bytes)
|
||||||
|
if isinstance(storage, ArtinixInMemoryStorage):
|
||||||
|
storage.data[ident] = BytesIO(data)
|
||||||
|
return
|
||||||
|
if isinstance(storage, ArtinixFileStorage):
|
||||||
|
with open(storage.basedir / ident, 'wb') as fp:
|
||||||
|
fp.write(data)
|
||||||
|
return
|
||||||
|
raise NotImplementedError("Forgot a case")
|
||||||
|
|
||||||
|
def writer(self, storage, ident):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def reader(self, storage, ident):
|
||||||
|
return None
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
from typing import Any, Iterable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
__all__ = ('ArtinixInMemoryStorage', 'ArtinixFileStorage')
|
||||||
|
|
||||||
|
class ArtinixInMemoryStorage:
|
||||||
|
def __init__(self):
|
||||||
|
self.data: dict[str, Any] = {}
|
||||||
|
|
||||||
|
def members(self) -> Iterable[str]:
|
||||||
|
return self.data.keys()
|
||||||
|
|
||||||
|
def has(self, ident: str) -> bool:
|
||||||
|
return ident in self.data
|
||||||
|
|
||||||
|
class ArtinixFileStorage:
|
||||||
|
def __init__(self, basedir: Path):
|
||||||
|
self.basedir = basedir
|
||||||
|
|
||||||
|
def members(self) -> Iterable[str]:
|
||||||
|
return [x.name for x in self.basedir.iterdir()]
|
||||||
|
|
||||||
|
def has(self, ident: str) -> bool:
|
||||||
|
return (self.basedir / ident).exists()
|
||||||
|
|
@ -0,0 +1,55 @@
|
||||||
|
#!@interpreter@
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from artinix_python import *
|
||||||
|
|
||||||
|
#
|
||||||
|
# Entity Classes
|
||||||
|
#
|
||||||
|
|
||||||
|
entity_classes = []
|
||||||
|
@entityClassSetup@
|
||||||
|
|
||||||
|
#
|
||||||
|
# Repositories
|
||||||
|
#
|
||||||
|
|
||||||
|
repositories = []
|
||||||
|
@repositorySetup@
|
||||||
|
|
||||||
|
#
|
||||||
|
# Functions
|
||||||
|
#
|
||||||
|
|
||||||
|
functions = []
|
||||||
|
@functionSetup@
|
||||||
|
|
||||||
|
#
|
||||||
|
# Bindings
|
||||||
|
#
|
||||||
|
|
||||||
|
bindings = []
|
||||||
|
@bindingSetup@
|
||||||
|
|
||||||
|
#
|
||||||
|
# Datastore setup
|
||||||
|
#
|
||||||
|
|
||||||
|
datastore = {}
|
||||||
|
@datastoreSetup@
|
||||||
|
|
||||||
|
#
|
||||||
|
# Go!
|
||||||
|
#
|
||||||
|
|
||||||
|
runtime = ArtinixPythonRuntime(
|
||||||
|
argv0=sys.argv[0],
|
||||||
|
default_setup = @allowDefault@,
|
||||||
|
manual_setup = @allowManual@,
|
||||||
|
bindings=bindings,
|
||||||
|
datastore=datastore,
|
||||||
|
)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
runtime.main(sys.argv[1:])
|
||||||
|
|
@ -0,0 +1,25 @@
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
}:
|
||||||
|
with lib;
|
||||||
|
mkModule (self: {
|
||||||
|
dtypes = mkModule (self': {
|
||||||
|
blob = mkRepoType {};
|
||||||
|
json = mkRepoType {
|
||||||
|
schema = {
|
||||||
|
int = { type = "int"; };
|
||||||
|
str = { type = "str"; };
|
||||||
|
listOf = ety: { type = "list"; inherit ety; };
|
||||||
|
dictOf = kty: vty: { type = "dict"; inherit kty vty; };
|
||||||
|
};
|
||||||
|
};
|
||||||
|
filesystem = mkRepoType {};
|
||||||
|
foreignKeyOf = foreignClass: mkRepoType { class = foreignClass; };
|
||||||
|
seqOf = ty: mkRepoType { inherit ty; };
|
||||||
|
tupleOf = children: mkRepoType { inherit children; };
|
||||||
|
});
|
||||||
|
repoAdapters = mkModule (self': {
|
||||||
|
seqToSpawn = mkAdapter { /* TODO */ };
|
||||||
|
tupleToRepos = mkAdapter { /* TODO */ };
|
||||||
|
});
|
||||||
|
})
|
||||||
Loading…
Reference in New Issue