diff --git a/Algorithmia/CLI.py b/Algorithmia/CLI.py index 02d27f8..3368915 100644 --- a/Algorithmia/CLI.py +++ b/Algorithmia/CLI.py @@ -2,10 +2,11 @@ import os from Algorithmia.errors import DataApiError from Algorithmia.algo_response import AlgoResponse +from Algorithmia.util import md5_for_file, md5_for_str import json, re, requests, six import toml import shutil - +from time import time class CLI: def __init__(self): @@ -101,7 +102,7 @@ def runalgo(self, options, client): elif (options.binary_file != None): # binary file with open(options.binary_file, "rb") as f: - algo_inputs = bytes(f.read()) + algo_input = bytes(f.read()) key = self.getAPIkey(options.profile) content = 'application/octet-stream' @@ -245,6 +246,31 @@ def cat(self, path, client): return result + # algo freeze + def freezeAlgo(self, client, manifest_path="model_manifest.json"): + if os.path.exists(manifest_path): + with open(manifest_path, 'r') as f: + manifest_file = json.load(f) + manifest_file['timestamp'] = str(time()) + required_files = manifest_file['required_files'] + optional_files = manifest_file['optional_files'] + for i in range(len(required_files)): + uri = required_files[i]['source_uri'] + local_file = client.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + for i in range(len(optional_files)): + uri = required_files[i]['source_uri'] + local_file = client.file(uri).getFile(as_path=True) + md5_checksum = md5_for_file(local_file) + required_files[i]['md5_checksum'] = md5_checksum + lock_md5_checksum = md5_for_str(str(manifest_file)) + manifest_file['lock_checksum'] = lock_md5_checksum + with open('model_manifest.json.freeze', 'w') as f: + json.dump(manifest_file, f) + else: + print("Expected to find a model_manifest.json file, none was discovered in working directory") + # algo cp def cp(self, src, dest, client): diff --git a/Algorithmia/__main__.py b/Algorithmia/__main__.py index c45e5fb..9e67c5c 100644 --- a/Algorithmia/__main__.py +++ b/Algorithmia/__main__.py @@ -108,7 +108,7 @@ def main(): parser_cat.add_argument('--profile', action = 'store', type = str, default = 'default') #sub parser for getting environment template - parser_template = subparsers.add_parser('template',help='template downloads an environment template to the destination') + parser_template = subparsers.add_parser('template', help='template downloads an environment template to the destination') parser_template.add_argument('envid',help='environment specification id') parser_template.add_argument('dest',help='destination for template download') @@ -130,8 +130,12 @@ def main(): subparsers.add_parser('help') parser.add_argument('--profile', action = 'store', type = str, default = 'default') + #sub parser for freeze + subparsers.add_parser('freeze', help="freezes a model_manifest.json file into a model_manifest.json.freeze") + args = parser.parse_args() + #run auth before trying to create a client if args.cmd == 'auth': @@ -215,6 +219,9 @@ def main(): elif args.cmd == 'builds': print(CLI().getBuildLogs(args.user, args.algo, client)) + elif args.cmd == "freeze": + print(CLI().freezeAlgo(client)) + else: parser.parse_args(['-h']) diff --git a/Algorithmia/util.py b/Algorithmia/util.py index 382b586..92aa3b3 100644 --- a/Algorithmia/util.py +++ b/Algorithmia/util.py @@ -1,8 +1,10 @@ import re -from Algorithmia.errors import DataApiError +import hashlib FNAME_MATCH = re.compile(r'/([^/]+)$') # From the last slash to the end of the string -PREFIX = re.compile(r'([^:]+://)(/)?(.+)') # Check for a prefix like data:// +PREFIX = re.compile(r'([^:]+://)(/)?(.+)') # Check for a prefix like data:// + + def getParentAndBase(path): match = PREFIX.match(path) if match is None: @@ -26,7 +28,22 @@ def getParentAndBase(path): parent_path = '{prefix}{uri}'.format(prefix=prefix, uri='/'.join(parts[:-1])) return parent_path, parts[-1] + def pathJoin(parent, base): if parent.endswith('/'): return parent + base return parent + '/' + base + + +def md5_for_file(fname): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return str(hash_md5.hexdigest()) + + +def md5_for_str(content): + hash_md5 = hashlib.md5() + hash_md5.update(content.encode()) + return str(hash_md5.hexdigest())