from includes import api, helpers

storage = helpers.read_text_file_as_array('links-storage')
pages = api.get_pages()

for index, page_url in enumerate(pages):

    print('page {} of {}'.format(index + 1, len(pages)))

    for post in api.get_posts(page_url):

        print('getting post info from: {}'.format(post['title']))

        links = api.get_post_raw_links(post['href'])

        for link in links:
            if link in storage:
                continue

            helpers.append_file('raw-links', link)
            helpers.append_file('raw-links-storage', link)

print('process finished.')