Files
UIX-R/make_release.py

176 lines
6.2 KiB
Python
Executable File

#!/usr/bin/env python3
import csv
import glob
from multiprocessing import Process, Queue, current_process
from nif_walker import walk_nif
import os
import sys
import tarfile
RELEASE_PATH = "uixr-assets-le-1.3.tar"
ASSET_PATH = "../../../Private"
def spinning_cursor():
while True:
for cursor in '|/-\\':
yield cursor
def get_filename(string):
return os.path.basename(string.replace("\\", "/"))
def insensitive_glob(pattern, recursive):
def either(c):
return '[%s%s]' % (c.lower(), c.upper()) if c.isalpha() else c
return glob.iglob(''.join(map(either, pattern)), recursive=recursive)
def find_match(ap, rap):
asset_filename, asset_extension = os.path.splitext(ap.lower())
relative_asset_filename, relative_asset_extension = os.path.splitext(rap.lower())
if asset_filename == relative_asset_filename:
return True
else:
return False
def retrieve_assets_from_nif(nif_path):
# check that we have all related assets
assets = []
nif_assets = walk_nif(nif_path=nif_path, use_stdout=False)
for asset_string in nif_assets:
for asset_name in asset_string.split(', '):
assets.append(get_filename(asset_name).lower())
return assets
def worker(work_queue, done_queue):
spinner = spinning_cursor()
p = current_process()
for nif_path in iter(work_queue.get, 'STOP'):
sys.stdout.write("\r\b\033[K{0} [{1}][{2}][{3}]".format(
next(spinner), work_queue.qsize(), p.name, nif_path))
sys.stdout.flush()
assets = []
try:
# assets.append('DEADBEEF')
assets = retrieve_assets_from_nif(nif_path)
except Exception:
pass
done_queue.put((nif_path, assets))
done_queue.put('STOP')
return True
def main():
with open("Manifests/UIXR.manifest") as f:
uixr_data = csv.DictReader(f)
tar_ball = tarfile.open(RELEASE_PATH, "w:")
spinner = spinning_cursor()
manifest = []
additional_assets = []
additional_assets_nif = {}
# setup multi-processing job
workers = 8
processes = []
work_queue = Queue()
done_queue = Queue()
print("Gathering assets: ")
for row in uixr_data:
row_license = row.get('license').lower()
if row_license == 'cc0' or row_license == 'cc-by' or row_license == 'cc-by-nc':
sys.stdout.write("\r\b \033[K{0} [{1}]".format(next(spinner), row.get('asset')))
sys.stdout.flush()
file_path = os.path.join(ASSET_PATH, row.get('asset'))
if not os.path.exists(file_path):
print("WARNING: asset not found -> {0}".format(row.get('asset')))
continue
manifest.append(row.get('asset'))
tar_ball.add(file_path, row.get('asset'))
work_queue.put(file_path)
# let multiprocessing parse the nifs
print("\n\nParsing NIFs for additional sub-assets: ")
for i in range(workers):
p = Process(target=worker, args=(work_queue, done_queue))
processes.append(p)
work_queue.put('STOP')
p.start()
stops = 0
while True:
item = done_queue.get()
if isinstance(item, str) and item == 'STOP':
stops += 1
if stops == workers:
break
else:
# for worker_name, nif_name, nif_assets in iter(done_queue.get, 'STOP'):
nif_path, nif_assets = item
additional_assets += nif_assets
additional_assets_nif[nif_path] = nif_assets
print("\n\nFiltering assets.")
# remove duplicates
additional_assets = set(additional_assets)
# remove blank entry in set
if '' in additional_assets:
additional_assets.remove('')
# remove files listed in manifest, we already have them
for asset in manifest:
filename = os.path.basename(asset).lower()
if filename in additional_assets:
additional_assets.remove(filename)
# match assets back to NIF
additional_assets = list(additional_assets)
for i in range(len(additional_assets)):
asset = additional_assets[i]
nifs_found = []
for nif, assets in additional_assets_nif.items():
if asset in assets:
nifs_found.append(nif)
additional_assets[i] = (asset, [nif.replace(ASSET_PATH, '') for nif in nifs_found])
print("\n\nGathering sub-assets: ")
# iterate through all sub assets
for nif_asset, nifs in additional_assets:
found = False
na_filename, _ = os.path.splitext(nif_asset.lower())
na_filename += '.*'
glob_path = os.path.join(ASSET_PATH, "UIX", "**", na_filename)
for asset_path in insensitive_glob(glob_path, True):
found = True
relative_asset_path = os.path.relpath(os.path.realpath(asset_path), ASSET_PATH)
if relative_asset_path in tar_ball.getnames():
break # file already exists, skip
sys.stdout.write("\r\b\033[K{0} [{1}]".format(next(spinner), relative_asset_path))
sys.stdout.flush()
f.seek(0) # reset to beginning of csv file
for row in uixr_data:
if find_match(row.get('asset'), relative_asset_path):
row_license = row.get('license').lower()
if row_license == 'cc0':
break # good to go, break
elif row_license == 'cc-by':
break # good to go, break
elif row_license == 'cc-by-nc':
break # good to go, break
else:
print("\nWARNING: Non-CC license asset -> {0} ({1})\n".format(
relative_asset_path, nifs))
if not found:
print("\nWARNING: sub-asset not found -> {0} ({1})\n".format(nif_asset, nifs))
tar_ball.close()
if __name__ == "__main__":
main()