\n2<\/strong>. Takes first link and extracts HTML text from this link.
\n3<\/strong>. Extracts Google Analytics account number from HTML. The acconneunt number usually appears on web page code on the line, formatted like this : ‘_uacct = “UA-xxxxxxxxxx”;’ The script extracts UA- number using regular expression.
\n4<\/strong>. Extracts Google AdSense information from HTML text. AdSense information is displayed within \/* *\/ like below:<\/p>\n
google_ad_client = “pub-xxxxxxxxx”;
\n\/* 300×15, created 1\/20\/17 *\/
\ngoogle_ad_slot = “xxxxxxxx”;
\nHere ‘300×15, created 1\/20\/17’ is default ad name.<\/p>\n
5<\/strong>. Extracts all links from the same HTML text. Here are few examples how the script can be used:<\/p>\n Below you can find script source code and flow chart.<\/p>\n
\n6<\/strong>. Adds links to list. Links are added to list only if they are from specific website domain.
\n7<\/strong>. Outputs extracted information to csv file. The saved information contains url, GA account number and AdSense ad names that are on the page.
\n8<\/strong>. Repeats steps 2 – 6 while there are links to process.<\/p>\n\n
\r\n# -*- coding: utf-8 -*-\r\n\r\nimport urllib.request\r\nimport lxml.html\r\nimport csv\r\nimport time\r\nimport os\r\nimport re\r\nimport string\r\nimport requests\r\n\r\npath=\"C:\\\\Users\\\\Owner\\\\Desktop\\\\2017\"\r\n\r\nfilename = path + \"\\\\\" + \"urlsB.csv\" \r\n\r\nfilename_info_extracted= path + \"\\\\\" + \"urls_info_extracted.csv\"\r\n\r\nurls_to_skip =['search3.cgi']\r\n\r\ndef load_file(fn):\r\n start=0\r\n file_urls=[] \r\n with open(fn, encoding=\"utf8\" ) as f:\r\n csv_f = csv.reader(f)\r\n for i, row in enumerate(csv_f):\r\n if i >= start :\r\n file_urls.append (row)\r\n return file_urls\r\n\r\ndef save_extracted_url (fn, row):\r\n \r\n if (os.path.isfile(fn)):\r\n m=\"a\"\r\n else:\r\n m=\"w\"\r\n \r\n \r\n with open(fn, m, encoding=\"utf8\", newline='' ) as csvfile: \r\n fieldnames = ['url', 'GA', 'GS']\r\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\r\n if (m==\"w\"):\r\n writer.writeheader()\r\n writer.writerow(row)\r\n\r\n\r\nlinks_processed = []\r\n\r\nurlsA= load_file (filename)\r\nprint (\"Starting navigate...\")\r\n\r\nurl_ind=0 \r\ndone=False\r\nwhile not done:\r\n u=urlsA[url_ind] \r\n new_row={} \r\n print (u[0])\r\n print (u)\r\n \r\n try:\r\n connection = urllib.request.urlopen(u[0])\r\n print (u[0])\r\n print (\"connected\")\r\n dom = lxml.html.fromstring(connection.read())\r\n time.sleep( 12 )\r\n r = requests.get(u[0])\r\n\r\n # Get the text of the contents\r\n html_content = r.text\r\n \r\n \r\n pat = re.compile(r\"(\/\\*(.+?)\\*\/)+\", re.MULTILINE)\r\n if pat.search(html_content) != None:\r\n \r\n str=\"\"\r\n for match in pat.finditer(html_content):\r\n print (\"%s: %s\" % (match.start(), match.group(1)))\r\n str=str+\",\"+ match.group(1)\r\n new_row['GS'] = str\r\n \r\n pat1 = re.compile(r\"_uacct(.+?)google_ad_client\", re.MULTILINE)\r\n pat1 = re.compile(r\"_uacct(.+?)\\\"(.+?)\\\"\", re.MULTILINE)\r\n if pat1.search(html_content) != None:\r\n \r\n m=pat1.search(html_content)\r\n new_row['GA'] = m.group(2)\r\n \r\n \r\n links_processed.append (u) \r\n\r\n new_row['url'] = u[0] \r\n save_extracted_url (filename_info_extracted, new_row)\r\n url_ind=url_ind+1\r\n \r\n \r\n print (html_content)\r\n \r\n links=[]\r\n for link in dom.xpath('\/\/a\/@href'):\r\n \r\n a=link.split(\"?\") \r\n if \"lwebzem\" in a[0]:\r\n \r\n try:\r\n links.append (link)\r\n ind=string.find(link, \"?\")\r\n \r\n if ind >=0:\r\n print ( link[:ind])\r\n else :\r\n print (link)\r\n \r\n except :\r\n print (\"EXCP\" + link)\r\n \r\n skip = False \r\n if [link] in links_processed:\r\n print (\"SKIPPED \" + link)\r\n skip=True\r\n if link in urlsA:\r\n skip = True\r\n if urls_to_skip[0] in link:\r\n skip=True\r\n if not skip: \r\n urlsA.append ([link])\r\n \r\n \r\n except:\r\n url_ind=url_ind+1 \r\n if url_ind > len(urlsA) :\r\n done=True \r\n\r\n<\/code><\/pre>\n![\"\"](\"http:\/\/intelligentonlinetools.com\/blog\/wp-content\/uploads\/2017\/04\/extractingGAGS_info-130x300.jpg\")
<\/p>\n","protected":false},"excerpt":{"rendered":"