#!/usr/bin/env python3
import os
import sys
import shutil

NUM_MERGED_FILES = 16
FILE_LIST_PATH = "/tmp/files"  # file containing the list of file paths, one per line.
OUTPUT_BASENAME = "/tmp/merged_"  # will be appended with the index 0..15

def get_file_size(path):
    try:
        return os.stat(path).st_size
    except OSError as e:
        sys.stderr.write(f"Warning: Could not stat {path}: {e}\n")
        return 0

def assign_files_to_bins(file_infos, num_bins):
    # bins will be a list of tuples (total_size, list_of_file_paths)
    bins = [(0, []) for _ in range(num_bins)]
    # Process files in descending order by size for a better greedy result:
    for size, path in sorted(file_infos, key=lambda x: x[0], reverse=True):
        # Find the bin with the least total size
        min_index = min(range(num_bins), key=lambda i: bins[i][0])
        current_total, paths = bins[min_index]
        # Append this file to that bin
        bins[min_index] = (current_total + size, paths + [path])
    return bins

def merge_files(bin_files, output_filename):
    # Opens the output file and copies in each file assigned in the bin.
    with open(output_filename, "wb") as outfile:
        for in_path in bin_files:
            try:
                # Read and write in chunks.
                with open(in_path, "rb") as infile:
                    shutil.copyfileobj(infile, outfile)
            except OSError as e:
                sys.stderr.write(f"Warning: Could not merge {in_path}: {e}\n")

def main():
    # Read the file list from /tmp/files.
    if not os.path.exists(FILE_LIST_PATH):
        sys.stderr.write(f"Error: File {FILE_LIST_PATH} does not exist.\n")
        sys.exit(1)
    
    with open(FILE_LIST_PATH, "r") as f:
        file_paths = [line.strip() for line in f if line.strip()]

    if not file_paths:
        sys.stderr.write("Error: No file paths found in the file list.\n")
        sys.exit(1)

    # Get file sizes and prepare list of tuples: (size, path)
    file_infos = []
    for path in file_paths:
        size = get_file_size(path)
        file_infos.append((size, path))

    # Assign the files to bins trying to equalize total size.
    bins = assign_files_to_bins(file_infos, NUM_MERGED_FILES)

    # For debugging or logging, you might want to print the total sizes
    for i, (total_size, paths) in enumerate(bins):
        print(f"Bin {i}: {len(paths)} files, total {total_size} bytes.")

    # Merge the files for each bin into one output file.
    for i, (total_size, paths) in enumerate(bins):
        output_file = OUTPUT_BASENAME + str(i)
        print(f"Writing {len(paths)} files to {output_file} (approx. {total_size} bytes)")
        merge_files(paths, output_file)

if __name__ == "__main__":
    main()

#!/usr/bin/env python3
import os
import sys
import shutil

NUM_MERGED_FILES = 16
FILE_LIST_PATH = "/tmp/files"  # file containing the list of file paths, one per line.
OUTPUT_BASENAME = "/tmp/merged_"  # will be appended with the index 0..15

def get_file_size(path):
    try:
        return os.stat(path).st_size
    except OSError as e:
        sys.stderr.write(f"Warning: Could not stat {path}: {e}\n")
        return 0

def assign_files_to_bins(file_infos, num_bins):
    # bins will be a list of tuples (total_size, list_of_file_paths)
    bins = [(0, []) for _ in range(num_bins)]
    # Process files in descending order by size for a better greedy result:
    for size, path in sorted(file_infos, key=lambda x: x[0], reverse=True):
        # Find the bin with the least total size
        min_index = min(range(num_bins), key=lambda i: bins[i][0])
        current_total, paths = bins[min_index]
        # Append this file to that bin
        bins[min_index] = (current_total + size, paths + [path])
    return bins

def merge_files(bin_files, output_filename):
    # Opens the output file and copies in each file assigned in the bin.
    with open(output_filename, "wb") as outfile:
        for in_path in bin_files:
            try:
                # Read and write in chunks.
                with open(in_path, "rb") as infile:
                    shutil.copyfileobj(infile, outfile)
            except OSError as e:
                sys.stderr.write(f"Warning: Could not merge {in_path}: {e}\n")

def main():
    # Read the file list from /tmp/files.
    if not os.path.exists(FILE_LIST_PATH):
        sys.stderr.write(f"Error: File {FILE_LIST_PATH} does not exist.\n")
        sys.exit(1)
    
    with open(FILE_LIST_PATH, "r") as f:
        file_paths = [line.strip() for line in f if line.strip()]

    if not file_paths:
        sys.stderr.write("Error: No file paths found in the file list.\n")
        sys.exit(1)

    # Get file sizes and prepare list of tuples: (size, path)
    file_infos = []
    for path in file_paths:
        size = get_file_size(path)
        file_infos.append((size, path))

    # Assign the files to bins trying to equalize total size.
    bins = assign_files_to_bins(file_infos, NUM_MERGED_FILES)

    # For debugging or logging, you might want to print the total sizes
    for i, (total_size, paths) in enumerate(bins):
        print(f"Bin {i}: {len(paths)} files, total {total_size} bytes.")

    # Merge the files for each bin into one output file.
    for i, (total_size, paths) in enumerate(bins):
        output_file = OUTPUT_BASENAME + str(i)
        print(f"Writing {len(paths)} files to {output_file} (approx. {total_size} bytes)")
        merge_files(paths, output_file)

if __name__ == "__main__":
    main()

#!/usr/bin/env python3
import os

NUM_MERGED = 16
FILE_LIST_PATH = "/tmp/files"

def main():
    # Step 1: Read file paths
    with open(FILE_LIST_PATH, 'r') as f_list:
        paths = [line.strip() for line in f_list if line.strip()]
    if not paths:
        print("No files found in", FILE_LIST_PATH)
        return

    # Step 2: Get (size, path) for each file.
    files = []
    for path in paths:
        try:
            size = os.path.getsize(path)
        except OSError as e:
            print(f"Warning: Could not get size for {path}: {e}")
            size = 0
        files.append((size, path))

    # Sort files descending by file size (largest first)
    files.sort(key=lambda tup: tup[0], reverse=True)

    # Step 3: Greedy partitioning into NUM_MERGED bins.
    # bins is a list of tuples: (current_total_size, [list of file paths])
    bins = [(0, []) for _ in range(NUM_MERGED)]
    for size, path in files:
        # choose the bin with currently the smallest total size
        min_index = min(range(NUM_MERGED), key=lambda i: bins[i][0])
        total_size, file_list = bins[min_index]
        file_list.append(path)
        bins[min_index] = (total_size + size, file_list)

    # (Optional) print out the total bytes in each bin for debugging.
    for i, (total, file_list) in enumerate(bins):
        print(f"Bin {i}: {total} bytes from {len(file_list)} files.")

    # Step 4: Write out each merged file.
    for i, (total, file_list) in enumerate(bins):
        output_path = f"/tmp/merged_{i}"
        print(f"Writing {len(file_list)} files to {output_path} (≈{total} bytes)")
        with open(output_path, 'wb') as out_f:
            # For each file assigned to this bin
            for fpath in file_list:
                try:
                    with open(fpath, 'rb') as in_f:
                        # Copy in blocks to avoid loading whole files into memory.
                        while True:
                            chunk = in_f.read(4096)
                            if not chunk:
                                break
                            out_f.write(chunk)
                except Exception as e:
                    print(f"Error reading {fpath}: {e}")

if __name__ == '__main__':
    main()

#!/usr/bin/env python3
import os

NUM_MERGED = 16
FILE_LIST_PATH = "/tmp/files"

def main():
    # Step 1: Read file paths
    with open(FILE_LIST_PATH, 'r') as f_list:
        paths = [line.strip() for line in f_list if line.strip()]
    if not paths:
        print("No files found in", FILE_LIST_PATH)
        return

    # Step 2: Get (size, path) for each file.
    files = []
    for path in paths:
        try:
            size = os.path.getsize(path)
        except OSError as e:
            print(f"Warning: Could not get size for {path}: {e}")
            size = 0
        files.append((size, path))

    # Sort files descending by file size (largest first)
    files.sort(key=lambda tup: tup[0], reverse=True)

    # Step 3: Greedy partitioning into NUM_MERGED bins.
    # bins is a list of tuples: (current_total_size, [list of file paths])
    bins = [(0, []) for _ in range(NUM_MERGED)]
    for size, path in files:
        # choose the bin with currently the smallest total size
        min_index = min(range(NUM_MERGED), key=lambda i: bins[i][0])
        total_size, file_list = bins[min_index]
        file_list.append(path)
        bins[min_index] = (total_size + size, file_list)

    # (Optional) print out the total bytes in each bin for debugging.
    for i, (total, file_list) in enumerate(bins):
        print(f"Bin {i}: {total} bytes from {len(file_list)} files.")

    # Step 4: Write out each merged file.
    for i, (total, file_list) in enumerate(bins):
        output_path = f"/tmp/merged_{i}"
        print(f"Writing {len(file_list)} files to {output_path} (≈{total} bytes)")
        with open(output_path, 'wb') as out_f:
            # For each file assigned to this bin
            for fpath in file_list:
                try:
                    with open(fpath, 'rb') as in_f:
                        # Copy in blocks to avoid loading whole files into memory.
                        while True:
                            chunk = in_f.read(4096)
                            if not chunk:
                                break
                            out_f.write(chunk)
                except Exception as e:
                    print(f"Error reading {fpath}: {e}")

if __name__ == '__main__':
    main()