commit ef53cbf40e8e84804f7af69937530cd9fb69f6d6 Author: Tony Date: Wed Jun 22 11:44:21 2022 -0500 Initial commit diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d4a2c44 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# http://editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +insert_final_newline = true +charset = utf-8 +end_of_line = lf + +[*.bat] +indent_style = tab +end_of_line = crlf + +[LICENSE] +insert_final_newline = false + +[Makefile] +indent_style = tab diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..d92a249 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,15 @@ +* gamescrapyr version: +* Python version: +* Operating System: + +### Description + +Describe what you were trying to get done. +Tell us what happened, what went wrong, and what you expected to happen. + +### What I Did + +``` +Paste the command(s) you ran and the output. +If there was a crash, please include the traceback here. +``` diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c915d1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,106 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +# IDE settings +.vscode/ +.idea/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..aaba487 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,15 @@ +# Config file for automatic testing at travis-ci.com + +language: python +python: + - 3.8 + - 3.7 + - 3.6 + +# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors +install: pip install -U tox-travis + +# Command to run tests, e.g. python setup.py test +script: tox + + diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..8392077 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,13 @@ +======= +Credits +======= + +Development Lead +---------------- + +* Anthony Correa + +Contributors +------------ + +None yet. Why not be the first? diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..e915e6f --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,128 @@ +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/anthonyscorrea/gamescrapyr/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +gamescrapyr could always use more documentation, whether as part of the +official gamescrapyr docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/anthonyscorrea/gamescrapyr/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `gamescrapyr` for local development. + +1. Fork the `gamescrapyr` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:your_name_here/gamescrapyr.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv gamescrapyr + $ cd gamescrapyr/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the + tests, including testing other Python versions with tox:: + + $ flake8 gamescrapyr tests + $ python setup.py test or pytest + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 3.5, 3.6, 3.7 and 3.8, and for PyPy. Check + https://travis-ci.com/anthonyscorrea/gamescrapyr/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + + + $ python -m unittest tests.test_gamescrapyr + +Deploying +--------- + +A reminder for the maintainers on how to deploy. +Make sure all your changes are committed (including an entry in HISTORY.rst). +Then run:: + +$ bump2version patch # possible: major / minor / patch +$ git push +$ git push --tags + +Travis will then deploy to PyPI if tests pass. diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..c1f536a --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.1.0 (2022-06-21) +------------------ + +* First release on PyPI. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f6a3c28 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2022, Anthony Correa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..965b2dd --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include AUTHORS.rst +include CONTRIBUTING.rst +include HISTORY.rst +include LICENSE +include README.rst + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] + +recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..cd6ccbc --- /dev/null +++ b/Makefile @@ -0,0 +1,89 @@ +.PHONY: clean clean-build clean-pyc clean-test coverage dist docs help install lint lint/flake8 lint/black +.DEFAULT_GOAL := help + +define BROWSER_PYSCRIPT +import os, webbrowser, sys + +from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT + +BROWSER := python -c "$$BROWSER_PYSCRIPT" + +help: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + rm -fr .pytest_cache + +lint/flake8: ## check style with flake8 + flake8 gamescrapyr tests +lint/black: ## check style with black + black --check gamescrapyr tests + +lint: lint/flake8 lint/black ## check style + +test: ## run tests quickly with the default Python + python setup.py test + +test-all: ## run tests on every Python version with tox + tox + +coverage: ## check code coverage quickly with the default Python + coverage run --source gamescrapyr setup.py test + coverage report -m + coverage html + $(BROWSER) htmlcov/index.html + +docs: ## generate Sphinx HTML documentation, including API docs + rm -f docs/gamescrapyr.rst + rm -f docs/modules.rst + sphinx-apidoc -o docs/ gamescrapyr + $(MAKE) -C docs clean + $(MAKE) -C docs html + $(BROWSER) docs/_build/html/index.html + +servedocs: docs ## compile the docs watching for changes + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + +release: dist ## package and upload a release + twine upload dist/* + +dist: clean ## builds source and wheel package + python setup.py sdist + python setup.py bdist_wheel + ls -l dist + +install: clean ## install the package to the active Python's site-packages + python setup.py install diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..9786880 --- /dev/null +++ b/README.rst @@ -0,0 +1,37 @@ +=========== +gamescrapyr +=========== + + +.. image:: https://img.shields.io/pypi/v/gamescrapyr.svg + :target: https://pypi.python.org/pypi/gamescrapyr + +.. image:: https://img.shields.io/travis/anthonyscorrea/gamescrapyr.svg + :target: https://travis-ci.com/anthonyscorrea/gamescrapyr + +.. image:: https://readthedocs.org/projects/gamescrapyr/badge/?version=latest + :target: https://gamescrapyr.readthedocs.io/en/latest/?version=latest + :alt: Documentation Status + + + + +Unofficial GameChanger scraper. + + +* Free software: MIT license +* Documentation: https://gamescrapyr.readthedocs.io. + + +Features +-------- + +* TODO + +Credits +------- + +This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template. + +.. _Cookiecutter: https://github.com/audreyr/cookiecutter +.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..01db7ee --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python -msphinx +SPHINXPROJ = gamescrapyr +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/authors.rst b/docs/authors.rst new file mode 100644 index 0000000..e122f91 --- /dev/null +++ b/docs/authors.rst @@ -0,0 +1 @@ +.. include:: ../AUTHORS.rst diff --git a/docs/conf.py b/docs/conf.py new file mode 100755 index 0000000..ca68183 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# +# gamescrapyr documentation build configuration file, created by +# sphinx-quickstart on Fri Jun 9 13:47:02 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another +# directory, add these directories to sys.path here. If the directory is +# relative to the documentation root, use os.path.abspath to make it +# absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +import gamescrapyr + +# -- General configuration --------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'gamescrapyr' +copyright = "2022, Anthony Correa" +author = "Anthony Correa" + +# The version info for the project you're documenting, acts as replacement +# for |version| and |release|, also used in various other places throughout +# the built documents. +# +# The short X.Y version. +version = gamescrapyr.__version__ +# The full version, including alpha/beta/rc tags. +release = gamescrapyr.__version__ + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a +# theme further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Options for HTMLHelp output --------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'gamescrapyrdoc' + + +# -- Options for LaTeX output ------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass +# [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'gamescrapyr.tex', + 'gamescrapyr Documentation', + 'Anthony Correa', 'manual'), +] + + +# -- Options for manual page output ------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'gamescrapyr', + 'gamescrapyr Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'gamescrapyr', + 'gamescrapyr Documentation', + author, + 'gamescrapyr', + 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 0000000..e582053 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../CONTRIBUTING.rst diff --git a/docs/history.rst b/docs/history.rst new file mode 100644 index 0000000..2506499 --- /dev/null +++ b/docs/history.rst @@ -0,0 +1 @@ +.. include:: ../HISTORY.rst diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..22661ba --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,20 @@ +Welcome to gamescrapyr's documentation! +====================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + readme + installation + usage + modules + contributing + authors + history + +Indices and tables +================== +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/installation.rst b/docs/installation.rst new file mode 100644 index 0000000..3872d21 --- /dev/null +++ b/docs/installation.rst @@ -0,0 +1,51 @@ +.. highlight:: shell + +============ +Installation +============ + + +Stable release +-------------- + +To install gamescrapyr, run this command in your terminal: + +.. code-block:: console + + $ pip install gamescrapyr + +This is the preferred method to install gamescrapyr, as it will always install the most recent stable release. + +If you don't have `pip`_ installed, this `Python installation guide`_ can guide +you through the process. + +.. _pip: https://pip.pypa.io +.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ + + +From sources +------------ + +The sources for gamescrapyr can be downloaded from the `Github repo`_. + +You can either clone the public repository: + +.. code-block:: console + + $ git clone git://github.com/anthonyscorrea/gamescrapyr + +Or download the `tarball`_: + +.. code-block:: console + + $ curl -OJL https://github.com/anthonyscorrea/gamescrapyr/tarball/master + +Once you have a copy of the source, you can install it with: + +.. code-block:: console + + $ python setup.py install + + +.. _Github repo: https://github.com/anthonyscorrea/gamescrapyr +.. _tarball: https://github.com/anthonyscorrea/gamescrapyr/tarball/master diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..4dc2679 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=python -msphinx +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=gamescrapyr + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The Sphinx module was not found. Make sure you have Sphinx installed, + echo.then set the SPHINXBUILD environment variable to point to the full + echo.path of the 'sphinx-build' executable. Alternatively you may add the + echo.Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/docs/readme.rst b/docs/readme.rst new file mode 100644 index 0000000..72a3355 --- /dev/null +++ b/docs/readme.rst @@ -0,0 +1 @@ +.. include:: ../README.rst diff --git a/docs/usage.rst b/docs/usage.rst new file mode 100644 index 0000000..7734b85 --- /dev/null +++ b/docs/usage.rst @@ -0,0 +1,7 @@ +===== +Usage +===== + +To use gamescrapyr in a project:: + + import gamescrapyr diff --git a/gamescrapyr/__init__.py b/gamescrapyr/__init__.py new file mode 100644 index 0000000..0aa1a57 --- /dev/null +++ b/gamescrapyr/__init__.py @@ -0,0 +1,5 @@ +"""Top-level package for gamescrapyr.""" + +__author__ = """Anthony Correa""" +__email__ = 'a@correa.co' +__version__ = '0.1.0' diff --git a/gamescrapyr/gamescrapyr.py b/gamescrapyr/gamescrapyr.py new file mode 100644 index 0000000..edea15d --- /dev/null +++ b/gamescrapyr/gamescrapyr.py @@ -0,0 +1,225 @@ +"""Main module.""" +from os import getenv + +import requests +import json +import re +import csv +from bs4 import BeautifulSoup +import datetime +import pytz + + +class GameChangerClient(): + session = requests.Session() + url = "https://gc.com/t/{season_slug}/{team_slug}-{team_id}/{page}" + + def __init__(self, email, password): + r = requests.get("https://gc.com/login") + + payload = { + 'email': email, + 'password': password, + "csrfmiddlewaretoken": r.cookies.get("csrftoken") + } + + self.session.headers["referer"] = "https://gc.com/do-login" + + signed_in = self.session.post( + url="https://gc.com/do-login", + headers=self.session.headers, + cookies=r.cookies, + data=payload + ) + + if signed_in.status_code != 200: + raise Exception(f'Login not successful, return code of {signed_in.status_code} for {signed_in.reason}') + elif b"Sign In to GC Classic" in signed_in.content or b"Sign Out" not in signed_in.content: + raise Exception (f'Login not successful, returned content not as expected') + + self.session.headers.update( + { + "x-csrftoken": r.cookies.get("csrftoken"), + } + ) + + pass + + def _scrape_from_page_initialize_json(self, season_slug, team_id, team_slug, page): + resp = self.session.get( + self.url.format(season_slug=season_slug, team_id=team_id, team_slug=team_slug, page=page) + ) + initialize_page_json = re.search( + r'page.initialize\(\$.parseJSON\("(.*?)"\)', resp.content.decode("unicode_escape") + ) + m = initialize_page_json.group(1) + return json.loads(m) + + def get_roster(self, season_slug, team_id, team_slug): + data = self._scrape_from_page_initialize_json(season_slug, team_id, team_slug, "roster") + return data['roster'] + + def get_stats(self, season_slug, team_id, team_slug): + resp = self.session.get( + self.url.format( + page="stats/batting/Qualified/standard/csv", + season_slug = season_slug, team_id = team_id, team_slug = team_slug + ) + ) + + if ( + resp.status_code != 200 + or b"Please sign in or join to continue." in resp.content + ): + raise Exception("Stats fetch failed.") + + roster = self.get_roster(season_slug = season_slug, team_id = team_id, team_slug = team_slug) + + id_lookup = { + (p.get("fname"), p.get("lname")): p.get("player_id") + for p in roster + } + + decoded_content = resp.content.decode("utf-8") + + cr = csv.reader(decoded_content.splitlines(), delimiter=",") + my_list = list(cr) + player_keys = [ + (i, key) + for i, key in enumerate(my_list[1][: my_list[0].index("Offensive Stats")]) + ] + offensive_keys = [ + (i, key) + for i, key in enumerate( + my_list[1][ + my_list[0] + .index("Offensive Stats"): my_list[0] + .index("Defensive Stats") + - 1 + ], + start=my_list[0].index("Offensive Stats"), + ) + ] + defensive_keys = [ + (i, key) + for i, key in enumerate( + my_list[1][my_list[0].index("Defensive Stats"):], + start=my_list[0].index("Defensive Stats"), + ) + ] + + stats = {} + for row in my_list[2:]: + player_keys + number, lname, fname = row[:3] + if number == "Team": + break + gamechanger_id = id_lookup[(fname, lname)] + stats[gamechanger_id] = { + "offensive": {k: row[i] for i, k in offensive_keys}, + "defensive": {k: row[i] for i, k in defensive_keys}, + } + + return stats + + def get_games(self, season_slug, team_id, team_slug): + page = self.session.get( + url=self.url.format( + page="schedule/games", + team_id=team_id, season_slug =season_slug, team_slug=team_slug), + ) + soup = BeautifulSoup(page.content, "html.parser") + game_elements = [r for r in soup.find_all("tr") if "game" in r.attrs.get("class")] + games = [] + for i, game_element in enumerate(game_elements): + game_slug = game_element.find("a").attrs.get("href").split("/")[1] + title = game_element.find("a").text + jslocaldate, jslocaltime_start, jslocaltime_arrival = ( + t.attrs.get("datetime") for t in game_element.findAll("time") + ) + games.append( + { + "id": game_element.attrs.get("data-id"), + "title": title, + "game_slug": game_slug, + "start": pytz.timezone("utc").localize( + datetime.datetime.fromisoformat(jslocaltime_start) + ), + } + ) + return games + + def submit_lineup(self, lineup, season_slug, team_slug, team_id): + headers = self.session.headers + headers.update( + { + "referer": self.url.format( + season_slug=season_slug, + team_slug=team_slug, + team_id=team_id, + page="lineup_edit", + ), + "x-csrftoken": self.session.cookies.get("csrftoken"), + "Content-Type": "application/x-www-form-urlencoded;", + } + ) + resp = self.session.post( + headers = headers, + cookies= self.session.cookies, + url= f"https://gc.com/do-save-lineup/{team_id}", + json={"lineup": lineup}, + ) + + if resp.status_code == 200 and resp.content == b"OK": + return resp + else: + raise requests.exceptions.RequestException( + f"Returned {resp.status_code} for {resp.reason}" + ) + + def get_game_stream(self, game_slug): + game_page = self.session.get( + cookies=self.session.cookies, + url=f"https://gc.com/{game_slug}", + ) + game_soup = BeautifulSoup(game_page.content, "html.parser") + data_push_url_rel = game_soup.find("body").attrs.get("data-push-url")[2:] + data_push_url = f"https://{data_push_url_rel}?sabertooth_aware=true" + stream_page = self.session.get( + cookies=self.session.cookies, url=data_push_url + ) + game_stream = json.loads(stream_page.content) + return game_stream + + def get_lineup(self, season_slug, team_slug, team_id): + resp = self.session.get( + self.url.format( + page="lineup", + season_slug=season_slug, team_id=team_id, team_slug=team_slug) + ) + + soup = BeautifulSoup(resp.content, "html.parser") + lineup_table = soup.find('table', {'id':'tbl_lineup'}) + lineup_rows = [r for r in lineup_table.findAll('tr')[1:] if r.text !='\nHas a Designated Hitter:\nPosition\n'] + lineup = [] + for row in lineup_rows: + order_element, player_element, position_element = row.findAll('td') + player_href = player_element.find(href=True)['href'] + player_id = player_href.split('-')[1] + order = order_element.text + d = { + 'player_id': player_id, + 'position': position_element.text + } + if order: + lineup.append(d) + elif not order: + dhs = [(i, r) for i, r in enumerate(lineup) if r['position']=='DH'] + if not dhs: + raise Exception("No DHs") + else: + dh_index, dh = dhs[0] + dh['forwhom'] = player_id + lineup.insert(dh_index+1, d) + + return lineup diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..0b055bc --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,12 @@ +pip==19.2.3 +bump2version==0.5.11 +wheel==0.33.6 +watchdog==0.9.0 +flake8==3.7.8 +tox==3.14.0 +coverage==4.5.4 +Sphinx==1.8.5 +twine==1.14.0 + + +black==21.7b0 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..62746d5 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,18 @@ +[bumpversion] +current_version = 0.1.0 +commit = True +tag = True + +[bumpversion:file:setup.py] +search = version='{current_version}' +replace = version='{new_version}' + +[bumpversion:file:gamescrapyr/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bdist_wheel] +universal = 1 + +[flake8] +exclude = docs diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f63204e --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +"""The setup script.""" + +from setuptools import setup, find_packages + +with open('README.rst') as readme_file: + readme = readme_file.read() + +with open('HISTORY.rst') as history_file: + history = history_file.read() + +requirements = [ ] + +test_requirements = [ ] + +setup( + author="Anthony Correa", + author_email='a@correa.co', + python_requires='>=3.6', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + ], + description="Unofficial GameChanger scraper.", + install_requires=requirements, + license="MIT license", + long_description=readme + '\n\n' + history, + include_package_data=True, + keywords='gamescrapyr', + name='gamescrapyr', + packages=find_packages(include=['gamescrapyr', 'gamescrapyr.*']), + test_suite='tests', + tests_require=test_requirements, + url='https://github.com/anthonyscorrea/gamescrapyr', + version='0.1.0', + zip_safe=False, +) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..7a8cd84 --- /dev/null +++ b/tox.ini @@ -0,0 +1,19 @@ +[tox] +envlist = py36, py37, py38, flake8 + +[travis] +python = + 3.8: py38 + 3.7: py37 + 3.6: py36 + +[testenv:flake8] +basepython = python +deps = flake8 +commands = flake8 gamescrapyr tests + +[testenv] +setenv = + PYTHONPATH = {toxinidir} + +commands = python setup.py test