mirror of
https://github.com/OpenMW/UIX-R.git
synced 2026-11-21 13:19:27 -10:00
176 lines
6.2 KiB
Python
Executable File
176 lines
6.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import csv
|
|
import glob
|
|
from multiprocessing import Process, Queue, current_process
|
|
from nif_walker import walk_nif
|
|
import os
|
|
import sys
|
|
import tarfile
|
|
|
|
RELEASE_PATH = "uixr-assets-le-1.3.tar"
|
|
ASSET_PATH = "../../../Private"
|
|
|
|
|
|
def spinning_cursor():
|
|
while True:
|
|
for cursor in '|/-\\':
|
|
yield cursor
|
|
|
|
|
|
def get_filename(string):
|
|
return os.path.basename(string.replace("\\", "/"))
|
|
|
|
|
|
def insensitive_glob(pattern, recursive):
|
|
def either(c):
|
|
return '[%s%s]' % (c.lower(), c.upper()) if c.isalpha() else c
|
|
return glob.iglob(''.join(map(either, pattern)), recursive=recursive)
|
|
|
|
|
|
def find_match(ap, rap):
|
|
asset_filename, asset_extension = os.path.splitext(ap.lower())
|
|
relative_asset_filename, relative_asset_extension = os.path.splitext(rap.lower())
|
|
|
|
if asset_filename == relative_asset_filename:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def retrieve_assets_from_nif(nif_path):
|
|
# check that we have all related assets
|
|
assets = []
|
|
nif_assets = walk_nif(nif_path=nif_path, use_stdout=False)
|
|
for asset_string in nif_assets:
|
|
for asset_name in asset_string.split(', '):
|
|
assets.append(get_filename(asset_name).lower())
|
|
return assets
|
|
|
|
|
|
def worker(work_queue, done_queue):
|
|
spinner = spinning_cursor()
|
|
p = current_process()
|
|
for nif_path in iter(work_queue.get, 'STOP'):
|
|
sys.stdout.write("\r\b\033[K{0} [{1}][{2}][{3}]".format(
|
|
next(spinner), work_queue.qsize(), p.name, nif_path))
|
|
sys.stdout.flush()
|
|
assets = []
|
|
try:
|
|
# assets.append('DEADBEEF')
|
|
assets = retrieve_assets_from_nif(nif_path)
|
|
except Exception:
|
|
pass
|
|
done_queue.put((nif_path, assets))
|
|
done_queue.put('STOP')
|
|
return True
|
|
|
|
|
|
def main():
|
|
with open("Manifests/UIXR.manifest") as f:
|
|
uixr_data = csv.DictReader(f)
|
|
tar_ball = tarfile.open(RELEASE_PATH, "w:")
|
|
spinner = spinning_cursor()
|
|
manifest = []
|
|
additional_assets = []
|
|
additional_assets_nif = {}
|
|
|
|
# setup multi-processing job
|
|
workers = 8
|
|
processes = []
|
|
work_queue = Queue()
|
|
done_queue = Queue()
|
|
|
|
print("Gathering assets: ")
|
|
for row in uixr_data:
|
|
row_license = row.get('license').lower()
|
|
if row_license == 'cc0' or row_license == 'cc-by' or row_license == 'cc-by-nc':
|
|
sys.stdout.write("\r\b \033[K{0} [{1}]".format(next(spinner), row.get('asset')))
|
|
sys.stdout.flush()
|
|
file_path = os.path.join(ASSET_PATH, row.get('asset'))
|
|
if not os.path.exists(file_path):
|
|
print("WARNING: asset not found -> {0}".format(row.get('asset')))
|
|
continue
|
|
manifest.append(row.get('asset'))
|
|
tar_ball.add(file_path, row.get('asset'))
|
|
|
|
work_queue.put(file_path)
|
|
|
|
# let multiprocessing parse the nifs
|
|
print("\n\nParsing NIFs for additional sub-assets: ")
|
|
for i in range(workers):
|
|
p = Process(target=worker, args=(work_queue, done_queue))
|
|
processes.append(p)
|
|
work_queue.put('STOP')
|
|
p.start()
|
|
|
|
stops = 0
|
|
while True:
|
|
item = done_queue.get()
|
|
if isinstance(item, str) and item == 'STOP':
|
|
stops += 1
|
|
if stops == workers:
|
|
break
|
|
else:
|
|
# for worker_name, nif_name, nif_assets in iter(done_queue.get, 'STOP'):
|
|
nif_path, nif_assets = item
|
|
additional_assets += nif_assets
|
|
additional_assets_nif[nif_path] = nif_assets
|
|
|
|
print("\n\nFiltering assets.")
|
|
# remove duplicates
|
|
additional_assets = set(additional_assets)
|
|
|
|
# remove blank entry in set
|
|
if '' in additional_assets:
|
|
additional_assets.remove('')
|
|
|
|
# remove files listed in manifest, we already have them
|
|
for asset in manifest:
|
|
filename = os.path.basename(asset).lower()
|
|
if filename in additional_assets:
|
|
additional_assets.remove(filename)
|
|
|
|
# match assets back to NIF
|
|
additional_assets = list(additional_assets)
|
|
for i in range(len(additional_assets)):
|
|
asset = additional_assets[i]
|
|
nifs_found = []
|
|
for nif, assets in additional_assets_nif.items():
|
|
if asset in assets:
|
|
nifs_found.append(nif)
|
|
additional_assets[i] = (asset, [nif.replace(ASSET_PATH, '') for nif in nifs_found])
|
|
|
|
print("\n\nGathering sub-assets: ")
|
|
# iterate through all sub assets
|
|
for nif_asset, nifs in additional_assets:
|
|
found = False
|
|
na_filename, _ = os.path.splitext(nif_asset.lower())
|
|
na_filename += '.*'
|
|
glob_path = os.path.join(ASSET_PATH, "UIX", "**", na_filename)
|
|
for asset_path in insensitive_glob(glob_path, True):
|
|
found = True
|
|
relative_asset_path = os.path.relpath(os.path.realpath(asset_path), ASSET_PATH)
|
|
if relative_asset_path in tar_ball.getnames():
|
|
break # file already exists, skip
|
|
sys.stdout.write("\r\b\033[K{0} [{1}]".format(next(spinner), relative_asset_path))
|
|
sys.stdout.flush()
|
|
f.seek(0) # reset to beginning of csv file
|
|
for row in uixr_data:
|
|
if find_match(row.get('asset'), relative_asset_path):
|
|
row_license = row.get('license').lower()
|
|
if row_license == 'cc0':
|
|
break # good to go, break
|
|
elif row_license == 'cc-by':
|
|
break # good to go, break
|
|
elif row_license == 'cc-by-nc':
|
|
break # good to go, break
|
|
else:
|
|
print("\nWARNING: Non-CC license asset -> {0} ({1})\n".format(
|
|
relative_asset_path, nifs))
|
|
if not found:
|
|
print("\nWARNING: sub-asset not found -> {0} ({1})\n".format(nif_asset, nifs))
|
|
tar_ball.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |