input_file = 'links.txt'
output_file = 'links_unique.txt'
log_file = 'duplicates.log'

seen = set()
unique_lines = []
duplicates = []

# خواندن فایل و تشخیص تکراری‌ها
with open(input_file, 'r', encoding='utf-8') as f:
    for line in f:
        line = line.rstrip('\n')  # حذف newline انتهای خط
        if line not in seen:
            seen.add(line)
            unique_lines.append(line)
        else:
            duplicates.append(line)

# نوشتن خطوط یکتا
with open(output_file, 'w', encoding='utf-8') as f:
    for line in unique_lines:
        f.write(line + '\n')

# نوشتن لاگ خطوط تکراری
with open(log_file, 'w', encoding='utf-8') as f:
    for line in duplicates:
        f.write(line + '\n')

print(f'{len(unique_lines)} unique lines written to {output_file}')
print(f'{len(duplicates)} duplicates written to {log_file}')