\n2. Downloading text content.<\/strong> The file with extracted links is opening and each link is visited and data (such as useful content no navigation, no advertisemet, html, title), are extracted using newspaper python module. This is running inside of function extract (url). Additionally extracted text content from each link is saving into memory list for LDA analysis on next step.
\n3. Text analyzing with LDA.<\/strong> Here thee script is preparing text data, doing actual LDA and outputting some results. Term, topic and probability also are saving in the file.<\/p>\n
Below are the figure for script flow and full python source code.<\/p>\n References<\/strong>![\"Program](\"http:\/\/intelligentonlinetools.com\/blog\/wp-content\/uploads\/2016\/12\/program-flow-300x247.png\")
\r\n# -*- coding: utf-8 -*-\r\nfrom newspaper import Article, Config\r\nimport os\r\nimport csv\r\nimport time\r\n\r\nimport urllib.request\r\nimport lxml.html\r\nimport re\r\n\r\nfrom nltk.tokenize import RegexpTokenizer\r\nfrom stop_words import get_stop_words\r\nfrom nltk.stem.porter import PorterStemmer\r\nfrom gensim import corpora \r\nimport gensim\r\n\r\n\r\n\r\n\r\nregex = re.compile(r'\\d\\d\\d\\d')\r\n\r\npath=\"C:\\\\Users\\\\Owner\\\\Python_2016\"\r\n\r\n#urlsA.csv file has the links for extracting web pages to visit\r\nfilename = path + \"\\\\\" + \"urlsA.csv\" \r\nfilename_urls_extracted= path + \"\\\\\" + \"urls_extracted.csv\"\r\n\r\ndef load_file(fn):\r\n start=0\r\n file_urls=[] \r\n with open(fn, encoding=\"utf8\" ) as f:\r\n csv_f = csv.reader(f)\r\n for i, row in enumerate(csv_f):\r\n if i >= start :\r\n file_urls.append (row)\r\n return file_urls\r\n\r\ndef save_extracted_url (fn, row):\r\n \r\n if (os.path.isfile(fn)):\r\n m=\"a\"\r\n else:\r\n m=\"w\"\r\n \r\n \r\n with open(fn, m, encoding=\"utf8\", newline='' ) as csvfile: \r\n fieldnames = ['url']\r\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\r\n if (m==\"w\"):\r\n writer.writeheader()\r\n writer.writerow(row)\r\n\r\nurlsA= load_file (filename)\r\nprint (\"Staring navigate...\")\r\nfor u in urlsA:\r\n print (u[0]) \r\n req = urllib.request.Request(u[0], headers={'User-Agent': 'Mozilla\/5.0'})\r\n connection = urllib.request.urlopen(req)\r\n print (\"connected\")\r\n dom = lxml.html.fromstring(connection.read())\r\n time.sleep( 7 )\r\n links=[]\r\n for link in dom.xpath('\/\/a\/@href'): \r\n try:\r\n \r\n links.append (link)\r\n except :\r\n print (\"EXCP\" + link)\r\n \r\n selected_links = list(filter(regex.search, links))\r\n \r\n\r\n link_data={} \r\n for link in selected_links:\r\n link_data['url'] = link\r\n save_extracted_url (filename_urls_extracted, link_data)\r\n\r\n\r\n\r\n#urls.csv file has the links for extracting content\r\nfilename = path + \"\\\\\" + \"urls.csv\" \r\n#data_from_urls.csv is file where extracted data is saved\r\nfilename_out= path + \"\\\\\" + \"data_from_urls.csv\"\r\n#below is the file where visited urls are saved\r\nfilename_urls_visited = path + \"\\\\\" + \"visited_urls.csv\"\r\n\r\n#load urls from file to memory\r\nurls= load_file (filename)\r\nvisited_urls=load_file (filename_urls_visited)\r\n\r\n\r\ndef save_to_file (fn, row):\r\n \r\n if (os.path.isfile(fn)):\r\n m=\"a\"\r\n else:\r\n m=\"w\"\r\n \r\n \r\n with open(fn, m, encoding=\"utf8\", newline='' ) as csvfile: \r\n fieldnames = ['url','authors', 'title', 'text', 'summary', 'keywords', 'publish_date', 'image', 'N']\r\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\r\n if (m==\"w\"):\r\n writer.writeheader()\r\n writer.writerow(row)\r\n \r\n\r\n\r\ndef save_visited_url (fn, row):\r\n \r\n if (os.path.isfile(fn)):\r\n m=\"a\"\r\n else:\r\n m=\"w\"\r\n \r\n \r\n with open(fn, m, encoding=\"utf8\", newline='' ) as csvfile: \r\n fieldnames = ['url']\r\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\r\n if (m==\"w\"):\r\n writer.writeheader()\r\n writer.writerow(row)\r\n \r\n#to save html to file we need to know prev. number of saved file\r\ndef get_last_number():\r\n path=\"C:\\\\Users\\\\Owner\\\\Desktop\\\\A\\\\Python_2016_A\" \r\n \r\n count=0\r\n for f in os.listdir(path):\r\n if f[-5:] == \".html\":\r\n count=count+1\r\n return (count) \r\n\r\n \r\nconfig = Config()\r\nconfig.keep_article_html = True\r\n\r\n\r\ndef extract(url):\r\n article = Article(url=url, config=config)\r\n article.download()\r\n time.sleep( 7 )\r\n article.parse()\r\n article.nlp()\r\n return dict(\r\n title=article.title,\r\n text=article.text,\r\n html=article.html,\r\n image=article.top_image,\r\n authors=article.authors,\r\n publish_date=article.publish_date,\r\n keywords=article.keywords,\r\n summary=article.summary,\r\n )\r\n\r\n\r\ndoc_set = []\r\n\r\nfor url in urls:\r\n newsp=extract (url[0])\r\n newsp['url'] = url\r\n \r\n next_number = get_last_number()\r\n next_number = next_number + 1\r\n newsp['N'] = str(next_number)+ \".html\"\r\n \r\n \r\n with open(str(next_number) + \".html\", \"w\", encoding='utf-8') as f:\r\n\t f.write(newsp['html'])\r\n print (\"HTML is saved to \" + str(next_number)+ \".html\")\r\n \r\n del newsp['html']\r\n \r\n u = {}\r\n u['url']=url\r\n doc_set.append (newsp['text'])\r\n save_to_file (filename_out, newsp)\r\n save_visited_url (filename_urls_visited, u)\r\n time.sleep( 4 )\r\n \r\n\r\n\r\n\r\ntokenizer = RegexpTokenizer(r'\\w+')\r\nen_stop = get_stop_words('en')\r\np_stemmer = PorterStemmer()\r\n \r\n\r\ntexts = []\r\n\r\n# loop through all documents\r\nfor i in doc_set:\r\n \r\n \r\n raw = i.lower()\r\n tokens = tokenizer.tokenize(raw)\r\n \r\n stopped_tokens = [i for i in tokens if not i in en_stop]\r\n \r\n stemmed_tokens = [p_stemmer.stem(i) for i in stopped_tokens]\r\n \r\n \r\n texts.append(stemmed_tokens)\r\n \r\nnum_topics = 2 \r\n\r\ndictionary = corpora.Dictionary(texts)\r\n \r\n\r\ncorpus = [dictionary.doc2bow(text) for text in texts]\r\nprint (corpus)\r\n\r\n# generate LDA model\r\nldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics=num_topics, id2word = dictionary, passes=20)\r\nprint (ldamodel)\r\n\r\nprint(ldamodel.print_topics(num_topics=3, num_words=3))\r\n\r\n#print topics containing term \"ai\"\r\nprint (ldamodel.get_term_topics(\"ai\", minimum_probability=None))\r\n\r\nprint (ldamodel.get_document_topics(corpus[0]))\r\n# Get Per-topic word probability matrix:\r\nK = ldamodel.num_topics\r\ntopicWordProbMat = ldamodel.print_topics(K)\r\nprint (topicWordProbMat)\r\n\r\n\r\n\r\nfn=\"topic_terms5.csv\"\r\nif (os.path.isfile(fn)):\r\n m=\"a\"\r\nelse:\r\n m=\"w\"\r\n\r\n# save topic, term, prob data in the file\r\nwith open(fn, m, encoding=\"utf8\", newline='' ) as csvfile: \r\n fieldnames = [\"topic_id\", \"term\", \"prob\"]\r\n writer = csv.DictWriter(csvfile, fieldnames=fieldnames)\r\n if (m==\"w\"):\r\n writer.writeheader()\r\n \r\n for topic_id in range(num_topics):\r\n term_probs = ldamodel.show_topic(topic_id, topn=6)\r\n for term, prob in term_probs:\r\n row={}\r\n row['topic_id']=topic_id\r\n row['prob']=prob\r\n row['term']=term\r\n writer.writerow(row)\r\n\r\n<\/code><\/pre>\n
\n1.Extracting Links from Web Pages Using Different Python Modules<\/a>
\n2.Web Content Extraction is Now Easier than Ever Using Python Scripting<\/a>
\n3.Latent Dirichlet allocation<\/a> Wikipedia
\n4.Latent Dirichlet Allocation<\/a>
\n5.Using Keyword Generation to refine Topic Models<\/a>
\n6. Beginners Guide to Topic Modeling in Python<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"