import urllib.parse as urlparse
from urllib.parse import parse_qs

def response(flow):
    global idx
    url_prefix = 'https://webcast.amemv.com/webcast/gift/list/'
    if flow.request.pretty_url.startswith(url_prefix):
        url = flow.request.pretty_url
        save_room_id(url)

def save_room_id(url):
    try:
        parsed = urlparse.urlparse(url)
        room_id = parse_qs(parsed.query)['room_id']
        with open('rooms.txt', 'a') as f:
            for data in room_id:
                f.write(data)
                f.write('\
')
    except:
        print('error')

# -*- coding: utf-8 -*-
from appium import webdriver
from time import sleep
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By

i=0

class DouYin(object):

    def __init__(self):
        # self.init_file()
        self.desired_caps={
        'platformName' : 'Android',
        'platformVersion' : '6.0.1',
        'deviceName' : '127.0.0.1:7555',
        'appPackage' : 'com.ss.android.ugc.aweme',
        'appActivity' : '.main.MainActivity'}
        self.driver=webdriver.Remote('http://localhost:47233/wd/hub', self.desired_caps)
        self.wait=WebDriverWait(self.driver, 60)
        self.get_permission()
        self.error_times=0
        # self.init_file()

    def swipe_up(self):
        self.driver.swipe(373, 1029, 373, 387)
        sleep(1)
        # self.driver.tap([(385, 471)])

    def zhibo(self):
        self.driver.tap([(45, 80)], 500)
        sleep(3)

    def get_permission(self):
        try:
            allow=self.wait.until(
                EC.presence_of_element_located((By.ID, 'com.ss.android.ugc.aweme:id/aps')))
            if allow is not None:
                allow.click()

            allow=self.wait.until(
                EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/permission_deny_button')))

            if allow is not None:
                allow.click()
            allow=self.wait.until(
                EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/permission_deny_button')))

            if allow is not None:
                allow.click()
            sleep(10)
            self.swipe_up()
            self.zhibo()


        except Exception as ex:
            print(ex)
            pass

    def click_up(self):
        ups_no=0

        while True:
            global i
            try:
                self.swipe_up()
                i +=1
                print(str(i))
            except Exception as err:
                print(err)
                self.error_times +=1
                if self.error_times < 5:
                    continue
                else:
                    raise Exception('Error happens 5 times, restart')

    def close_app(self):
        self.driver.close_app()

    def run(self):
        self.click_up()
        # self.close_app()

if __name__=='__main__':

    while True:
        try:

            douyin=DouYin()
            douyin.run()
        except Exception as error:
            print(error)

import requests
import json
import csv
from bs4 import BeautifulSoup
import re
import os
csv_file='douyin.csv'
csv_columns=['name', 'douyin_id', 'gender','followers','description', 'avatar']

def save():
    init_file()
    with open('rooms.txt','r') as txt_file:
        rooms=txt_file.read().split('\
')
        for room in rooms:
            save_one(room)

def save_one(room_id):
    url='https://webcast.amemv.com/webcast/reflow/' + room_id + '?utm_source=copy&utm_campaign=client_share&utm_medium=android&share_app_name=douyin'
    headers={
        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36',
    }
    response=requests.get(url, headers=headers).text
    soup=BeautifulSoup(response)
    a=soup.find_all('script')
    for t in a:
        if 'owner_user_id' in str(t):
            m=re.search('\\{.*\\}', str(t))
            j=m.group(0)
            j=json.loads(j)
            user=j['/webcast/reflow/:id']['room']['owner']
            save_ups(user)
            user=json.dumps(user)
            with open(f'user.json', 'a', encoding='utf-8-sig') as js:
                js.write(user)
                js.write('\
')

            break

def init_file():
    try:
        if os.path.exists(csv_file):
            return
        with open(csv_file, 'w',encoding='utf-8-sig') as csvfile:
            writer=csv.DictWriter(csvfile, fieldnames=csv_columns, delimiter=',', lineterminator='\
')
            writer.writeheader()


    except IOError as err:
        print(err)


def save_ups(user):
    up=dict()
    up['name']=user['nickname']
    up['douyin_id']=user['display_id']+'\	'
    up['gender']='女' if user['gender']==2 else '男'
    up['description']=user['signature']
    up['followers']=user['follow_info']['follower_count']
    up['avatar']=user['avatar_thumb']['url_list'][0]

    with open(csv_file, 'a', encoding='utf-8-sig') as douyin:
        writer=csv.DictWriter(douyin, fieldnames=csv_columns, delimiter=',', lineterminator='\
')
        writer.writerow(up)