make_release.py

#!/usr/bin/env python3
import csv
import glob
from multiprocessing import Process, Queue, current_process
from nif_walker import walk_nif
import os
import sys
import tarfile

RELEASE_PATH = "uixr-assets-le-1.3.tar"
ASSET_PATH = "../../../Private"


def spinning_cursor():
    while True:
        for cursor in '|/-\\':
            yield cursor


def get_filename(string):
    return os.path.basename(string.replace("\\", "/"))


def insensitive_glob(pattern, recursive):
    def either(c):
        return '[%s%s]' % (c.lower(), c.upper()) if c.isalpha() else c
    return glob.iglob(''.join(map(either, pattern)), recursive=recursive)


def find_match(ap, rap):
    asset_filename, asset_extension = os.path.splitext(ap.lower())
    relative_asset_filename, relative_asset_extension = os.path.splitext(rap.lower())

    if asset_filename == relative_asset_filename:
        return True
    else:
        return False


def retrieve_assets_from_nif(nif_path):
    # check that we have all related assets
    assets = []
    nif_assets = walk_nif(nif_path=nif_path, use_stdout=False)
    for asset_string in nif_assets:
        for asset_name in asset_string.split(', '):
            assets.append(get_filename(asset_name).lower())
    return assets


def worker(work_queue, done_queue):
    spinner = spinning_cursor()
    p = current_process()
    for nif_path in iter(work_queue.get, 'STOP'):
        sys.stdout.write("\r\b\033[K{0} [{1}][{2}][{3}]".format(
            next(spinner), work_queue.qsize(), p.name, nif_path))
        sys.stdout.flush()
        assets = []
        try:
            # assets.append('DEADBEEF')
            assets = retrieve_assets_from_nif(nif_path)
        except Exception:
            pass
        done_queue.put((nif_path, assets))
    done_queue.put('STOP')
    return True


def main():
    with open("Manifests/UIXR.manifest") as f:
        uixr_data = csv.DictReader(f)
        tar_ball = tarfile.open(RELEASE_PATH, "w:")
        spinner = spinning_cursor()
        manifest = []
        additional_assets = []
        additional_assets_nif = {}

        # setup multi-processing job
        workers = 8
        processes = []
        work_queue = Queue()
        done_queue = Queue()

        print("Gathering assets: ")
        for row in uixr_data:
            row_license = row.get('license').lower()
            if row_license == 'cc0' or row_license == 'cc-by' or row_license == 'cc-by-nc':
                sys.stdout.write("\r\b \033[K{0} [{1}]".format(next(spinner), row.get('asset')))
                sys.stdout.flush()
                file_path = os.path.join(ASSET_PATH, row.get('asset'))
                if not os.path.exists(file_path):
                    print("WARNING: asset not found -> {0}".format(row.get('asset')))
                    continue
                manifest.append(row.get('asset'))
                tar_ball.add(file_path, row.get('asset'))

                work_queue.put(file_path)

        # let multiprocessing parse the nifs
        print("\n\nParsing NIFs for additional sub-assets: ")
        for i in range(workers):
            p = Process(target=worker, args=(work_queue, done_queue))
            processes.append(p)
            work_queue.put('STOP')
            p.start()

        stops = 0
        while True:
            item = done_queue.get()
            if isinstance(item, str) and item == 'STOP':
                stops += 1
                if stops == workers:
                    break
            else:
                # for worker_name, nif_name, nif_assets in iter(done_queue.get, 'STOP'):
                nif_path, nif_assets = item
                additional_assets += nif_assets
                additional_assets_nif[nif_path] = nif_assets

        print("\n\nFiltering assets.")
        # remove duplicates
        additional_assets = set(additional_assets)

        # remove blank entry in set
        if '' in additional_assets:
            additional_assets.remove('')

        # remove files listed in manifest, we already have them
        for asset in manifest:
            filename = os.path.basename(asset).lower()
            if filename in additional_assets:
                additional_assets.remove(filename)

        # match assets back to NIF
        additional_assets = list(additional_assets)
        for i in range(len(additional_assets)):
            asset = additional_assets[i]
            nifs_found = []
            for nif, assets in additional_assets_nif.items():
                if asset in assets:
                    nifs_found.append(nif)
            additional_assets[i] = (asset, [nif.replace(ASSET_PATH, '') for nif in nifs_found])

        print("\n\nGathering sub-assets: ")
        # iterate through all sub assets
        for nif_asset, nifs in additional_assets:
            found = False
            na_filename, _ = os.path.splitext(nif_asset.lower())
            na_filename += '.*'
            glob_path = os.path.join(ASSET_PATH, "UIX", "**", na_filename)
            for asset_path in insensitive_glob(glob_path, True):
                found = True
                relative_asset_path = os.path.relpath(os.path.realpath(asset_path), ASSET_PATH)
                if relative_asset_path in tar_ball.getnames():
                    break  # file already exists, skip
                sys.stdout.write("\r\b\033[K{0} [{1}]".format(next(spinner), relative_asset_path))
                sys.stdout.flush()
                f.seek(0)   # reset to beginning of csv file
                for row in uixr_data:
                    if find_match(row.get('asset'), relative_asset_path):
                        row_license = row.get('license').lower()
                        if row_license == 'cc0':
                            break  # good to go, break
                        elif row_license == 'cc-by':
                            break  # good to go, break
                        elif row_license == 'cc-by-nc':
                            break  # good to go, break
                        else:
                            print("\nWARNING: Non-CC license asset -> {0} ({1})\n".format(
                                relative_asset_path, nifs))
            if not found:
                print("\nWARNING: sub-asset not found -> {0} ({1})\n".format(nif_asset, nifs))
        tar_ball.close()


if __name__ == "__main__":
    main()