utils/remove_duplicates.py

def remove_duplicates(items: list, key=lambda x: x, show_process=False):
    '''
    Remove duplicates from a list of items
    Args:
      items: List of items
      key: Function to get the key of the item
      show_process: Whether to show the process or not
    Returns:
      List: List of items without duplicates
    '''
    progress = lambda x, *, desc: x
    if show_process:
        import tqdm
        progress = tqdm.tqdm
    return list({key(item): item for item in progress(items, desc='Deduping...')}.values())