Project using python/Jobs scrapper
-
Extract reddit's posts and make export.html with Flask frameworkProject using python/Jobs scrapper 2020. 12. 21. 14:03
main.py // main.py from scrapperReddit import get_subreddits app = Flask("Job Scrapper", template_folder="./src/templates") db={} subreddits = [ "javascript", "reactjs", "reactnative", "programming", "css", "golang", "flutter", "rust", "django", ] @app.route('/') def index(): try: return render_template("home.html", subreddits=subreddits) except IOError: return redirect("/") @app.route('/reddit'..
-
export.html with Flask frameworkProject using python/Jobs scrapper 2020. 12. 21. 13:49
main.py from exporter import save_to_file @app.route('/export') def export(): try: word = request.args.get('word') if not word: print("not word") raise Exception() word = word.lower() jobs = db.get(word) if not jobs: print("not jobs") raise Exception() save_to_file(jobs, word) return redirect("/") except IOError: print("error") return redirect("/") search.html로 원하는 일자리들을 찾은 후에 해당 일자리를 .csv 파일로 다..
-
search.htmlProject using python/Jobs scrapper 2020. 12. 21. 13:44
main.py // main.py from flask import Flask, render_template, redirect, request from scrapperJobs import get_jobs db={} @app.route('/search') def search(): try: word = request.args.get('term') if word: word = word.lower() if word in db: jobs = db[word] else: jobs = get_jobs(word) db[word]=jobs else: redirect("/") return render_template("search.html", resultsNumber=len(jobs),searchingBy=word, jobs..
-
home.html with Flask frameworkProject using python/Jobs scrapper 2020. 12. 21. 13:16
Flask Introduction Flask는 쉽고 빠르게 web server를 만들 수 있는 python famework다. Installation pipenv install Flask Run flask // main.py from flask import Flask, render_template, redirect app = Flask("Job Scrapper", template_folder="./src/templates") @app.route('/') def index(): try: return render_template("home.html") except IOError: return redirect("/") app.run(host="127.0.0.1") app = Flask()를 통해서 Flask ..
-
Scrap remote.comProject using python/Jobs scrapper 2020. 12. 21. 11:53
scrapperRemote.py import requests from bs4 import BeautifulSoup headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36" } def extract_job(html): # title, company, location, link tds = html.find_all("td") link = tds[0].find("a")["href"] if not link: link = "" title = tds[1].find("h2").string if not title: ..
-
Scrap WeWorkRemotelyProject using python/Jobs scrapper 2020. 12. 21. 11:51
scrapperWWR.py import requests from bs4 import BeautifulSoup def extract_job(html): # title, company, location, link job_info_link = html.find_all('a') if len(job_info_link) > 1: job_info_link = job_info_link[1] else: job_info_link = job_info_link[0] link = f"https://weworkremotely.com/{job_info_link['href']}" job_info = job_info_link.find_all("span") company = job_info[0].get_text() title = job..
-
Extract jobs from Stack OverflowProject using python/Jobs scrapper 2020. 12. 21. 11:49
scrapperSO.py import requests from bs4 import BeautifulSoup def extract_job(html): # title, company, location, link title_link = html.find('h2').find('a',{'class':'s-link'}) title = title_link['title'] link = f"https://stackoverflow.com/jobs/{title_link['href']}" company, location = html.find('h3', {"class":"fs-body1"}).find_all('span', recursive=False) return { "title" : title, "link" : link, "..
-
get_last_page of Stack OverflowProject using python/Jobs scrapper 2020. 12. 21. 10:51
scrapperSO.py import requests from bs4 import BeautifulSoup def get_last_page(url): result = requests.get(url) soup = BeautifulSoup(result.text, "html.parser") pages = soup.find("div", {"class": "s-pagination"}).find_all('a') last_page = pages[-2].get_text(strip=True) return int(last_page) def get_SOJobs(word): url = f"https://stackoverflow.com/jobs?q={word}" last_page = get_last_page(url) print..