Initial commit
This commit is contained in:
21
.editorconfig
Normal file
21
.editorconfig
Normal file
@@ -0,0 +1,21 @@
|
||||
# http://editorconfig.org
|
||||
|
||||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
|
||||
[*.bat]
|
||||
indent_style = tab
|
||||
end_of_line = crlf
|
||||
|
||||
[LICENSE]
|
||||
insert_final_newline = false
|
||||
|
||||
[Makefile]
|
||||
indent_style = tab
|
||||
15
.github/ISSUE_TEMPLATE.md
vendored
Normal file
15
.github/ISSUE_TEMPLATE.md
vendored
Normal file
@@ -0,0 +1,15 @@
|
||||
* gamescrapyr version:
|
||||
* Python version:
|
||||
* Operating System:
|
||||
|
||||
### Description
|
||||
|
||||
Describe what you were trying to get done.
|
||||
Tell us what happened, what went wrong, and what you expected to happen.
|
||||
|
||||
### What I Did
|
||||
|
||||
```
|
||||
Paste the command(s) you ran and the output.
|
||||
If there was a crash, please include the traceback here.
|
||||
```
|
||||
106
.gitignore
vendored
Normal file
106
.gitignore
vendored
Normal file
@@ -0,0 +1,106 @@
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
env/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
.pytest_cache/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# dotenv
|
||||
.env
|
||||
|
||||
# virtualenv
|
||||
.venv
|
||||
venv/
|
||||
ENV/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
# IDE settings
|
||||
.vscode/
|
||||
.idea/
|
||||
15
.travis.yml
Normal file
15
.travis.yml
Normal file
@@ -0,0 +1,15 @@
|
||||
# Config file for automatic testing at travis-ci.com
|
||||
|
||||
language: python
|
||||
python:
|
||||
- 3.8
|
||||
- 3.7
|
||||
- 3.6
|
||||
|
||||
# Command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors
|
||||
install: pip install -U tox-travis
|
||||
|
||||
# Command to run tests, e.g. python setup.py test
|
||||
script: tox
|
||||
|
||||
|
||||
13
AUTHORS.rst
Normal file
13
AUTHORS.rst
Normal file
@@ -0,0 +1,13 @@
|
||||
=======
|
||||
Credits
|
||||
=======
|
||||
|
||||
Development Lead
|
||||
----------------
|
||||
|
||||
* Anthony Correa <a@correa.co>
|
||||
|
||||
Contributors
|
||||
------------
|
||||
|
||||
None yet. Why not be the first?
|
||||
128
CONTRIBUTING.rst
Normal file
128
CONTRIBUTING.rst
Normal file
@@ -0,0 +1,128 @@
|
||||
.. highlight:: shell
|
||||
|
||||
============
|
||||
Contributing
|
||||
============
|
||||
|
||||
Contributions are welcome, and they are greatly appreciated! Every little bit
|
||||
helps, and credit will always be given.
|
||||
|
||||
You can contribute in many ways:
|
||||
|
||||
Types of Contributions
|
||||
----------------------
|
||||
|
||||
Report Bugs
|
||||
~~~~~~~~~~~
|
||||
|
||||
Report bugs at https://github.com/anthonyscorrea/gamescrapyr/issues.
|
||||
|
||||
If you are reporting a bug, please include:
|
||||
|
||||
* Your operating system name and version.
|
||||
* Any details about your local setup that might be helpful in troubleshooting.
|
||||
* Detailed steps to reproduce the bug.
|
||||
|
||||
Fix Bugs
|
||||
~~~~~~~~
|
||||
|
||||
Look through the GitHub issues for bugs. Anything tagged with "bug" and "help
|
||||
wanted" is open to whoever wants to implement it.
|
||||
|
||||
Implement Features
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Look through the GitHub issues for features. Anything tagged with "enhancement"
|
||||
and "help wanted" is open to whoever wants to implement it.
|
||||
|
||||
Write Documentation
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
gamescrapyr could always use more documentation, whether as part of the
|
||||
official gamescrapyr docs, in docstrings, or even on the web in blog posts,
|
||||
articles, and such.
|
||||
|
||||
Submit Feedback
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
The best way to send feedback is to file an issue at https://github.com/anthonyscorrea/gamescrapyr/issues.
|
||||
|
||||
If you are proposing a feature:
|
||||
|
||||
* Explain in detail how it would work.
|
||||
* Keep the scope as narrow as possible, to make it easier to implement.
|
||||
* Remember that this is a volunteer-driven project, and that contributions
|
||||
are welcome :)
|
||||
|
||||
Get Started!
|
||||
------------
|
||||
|
||||
Ready to contribute? Here's how to set up `gamescrapyr` for local development.
|
||||
|
||||
1. Fork the `gamescrapyr` repo on GitHub.
|
||||
2. Clone your fork locally::
|
||||
|
||||
$ git clone git@github.com:your_name_here/gamescrapyr.git
|
||||
|
||||
3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development::
|
||||
|
||||
$ mkvirtualenv gamescrapyr
|
||||
$ cd gamescrapyr/
|
||||
$ python setup.py develop
|
||||
|
||||
4. Create a branch for local development::
|
||||
|
||||
$ git checkout -b name-of-your-bugfix-or-feature
|
||||
|
||||
Now you can make your changes locally.
|
||||
|
||||
5. When you're done making changes, check that your changes pass flake8 and the
|
||||
tests, including testing other Python versions with tox::
|
||||
|
||||
$ flake8 gamescrapyr tests
|
||||
$ python setup.py test or pytest
|
||||
$ tox
|
||||
|
||||
To get flake8 and tox, just pip install them into your virtualenv.
|
||||
|
||||
6. Commit your changes and push your branch to GitHub::
|
||||
|
||||
$ git add .
|
||||
$ git commit -m "Your detailed description of your changes."
|
||||
$ git push origin name-of-your-bugfix-or-feature
|
||||
|
||||
7. Submit a pull request through the GitHub website.
|
||||
|
||||
Pull Request Guidelines
|
||||
-----------------------
|
||||
|
||||
Before you submit a pull request, check that it meets these guidelines:
|
||||
|
||||
1. The pull request should include tests.
|
||||
2. If the pull request adds functionality, the docs should be updated. Put
|
||||
your new functionality into a function with a docstring, and add the
|
||||
feature to the list in README.rst.
|
||||
3. The pull request should work for Python 3.5, 3.6, 3.7 and 3.8, and for PyPy. Check
|
||||
https://travis-ci.com/anthonyscorrea/gamescrapyr/pull_requests
|
||||
and make sure that the tests pass for all supported Python versions.
|
||||
|
||||
Tips
|
||||
----
|
||||
|
||||
To run a subset of tests::
|
||||
|
||||
|
||||
$ python -m unittest tests.test_gamescrapyr
|
||||
|
||||
Deploying
|
||||
---------
|
||||
|
||||
A reminder for the maintainers on how to deploy.
|
||||
Make sure all your changes are committed (including an entry in HISTORY.rst).
|
||||
Then run::
|
||||
|
||||
$ bump2version patch # possible: major / minor / patch
|
||||
$ git push
|
||||
$ git push --tags
|
||||
|
||||
Travis will then deploy to PyPI if tests pass.
|
||||
8
HISTORY.rst
Normal file
8
HISTORY.rst
Normal file
@@ -0,0 +1,8 @@
|
||||
=======
|
||||
History
|
||||
=======
|
||||
|
||||
0.1.0 (2022-06-21)
|
||||
------------------
|
||||
|
||||
* First release on PyPI.
|
||||
22
LICENSE
Normal file
22
LICENSE
Normal file
@@ -0,0 +1,22 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022, Anthony Correa
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
11
MANIFEST.in
Normal file
11
MANIFEST.in
Normal file
@@ -0,0 +1,11 @@
|
||||
include AUTHORS.rst
|
||||
include CONTRIBUTING.rst
|
||||
include HISTORY.rst
|
||||
include LICENSE
|
||||
include README.rst
|
||||
|
||||
recursive-include tests *
|
||||
recursive-exclude * __pycache__
|
||||
recursive-exclude * *.py[co]
|
||||
|
||||
recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif
|
||||
89
Makefile
Normal file
89
Makefile
Normal file
@@ -0,0 +1,89 @@
|
||||
.PHONY: clean clean-build clean-pyc clean-test coverage dist docs help install lint lint/flake8 lint/black
|
||||
.DEFAULT_GOAL := help
|
||||
|
||||
define BROWSER_PYSCRIPT
|
||||
import os, webbrowser, sys
|
||||
|
||||
from urllib.request import pathname2url
|
||||
|
||||
webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
|
||||
endef
|
||||
export BROWSER_PYSCRIPT
|
||||
|
||||
define PRINT_HELP_PYSCRIPT
|
||||
import re, sys
|
||||
|
||||
for line in sys.stdin:
|
||||
match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
|
||||
if match:
|
||||
target, help = match.groups()
|
||||
print("%-20s %s" % (target, help))
|
||||
endef
|
||||
export PRINT_HELP_PYSCRIPT
|
||||
|
||||
BROWSER := python -c "$$BROWSER_PYSCRIPT"
|
||||
|
||||
help:
|
||||
@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
|
||||
|
||||
clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
|
||||
|
||||
clean-build: ## remove build artifacts
|
||||
rm -fr build/
|
||||
rm -fr dist/
|
||||
rm -fr .eggs/
|
||||
find . -name '*.egg-info' -exec rm -fr {} +
|
||||
find . -name '*.egg' -exec rm -f {} +
|
||||
|
||||
clean-pyc: ## remove Python file artifacts
|
||||
find . -name '*.pyc' -exec rm -f {} +
|
||||
find . -name '*.pyo' -exec rm -f {} +
|
||||
find . -name '*~' -exec rm -f {} +
|
||||
find . -name '__pycache__' -exec rm -fr {} +
|
||||
|
||||
clean-test: ## remove test and coverage artifacts
|
||||
rm -fr .tox/
|
||||
rm -f .coverage
|
||||
rm -fr htmlcov/
|
||||
rm -fr .pytest_cache
|
||||
|
||||
lint/flake8: ## check style with flake8
|
||||
flake8 gamescrapyr tests
|
||||
lint/black: ## check style with black
|
||||
black --check gamescrapyr tests
|
||||
|
||||
lint: lint/flake8 lint/black ## check style
|
||||
|
||||
test: ## run tests quickly with the default Python
|
||||
python setup.py test
|
||||
|
||||
test-all: ## run tests on every Python version with tox
|
||||
tox
|
||||
|
||||
coverage: ## check code coverage quickly with the default Python
|
||||
coverage run --source gamescrapyr setup.py test
|
||||
coverage report -m
|
||||
coverage html
|
||||
$(BROWSER) htmlcov/index.html
|
||||
|
||||
docs: ## generate Sphinx HTML documentation, including API docs
|
||||
rm -f docs/gamescrapyr.rst
|
||||
rm -f docs/modules.rst
|
||||
sphinx-apidoc -o docs/ gamescrapyr
|
||||
$(MAKE) -C docs clean
|
||||
$(MAKE) -C docs html
|
||||
$(BROWSER) docs/_build/html/index.html
|
||||
|
||||
servedocs: docs ## compile the docs watching for changes
|
||||
watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
|
||||
|
||||
release: dist ## package and upload a release
|
||||
twine upload dist/*
|
||||
|
||||
dist: clean ## builds source and wheel package
|
||||
python setup.py sdist
|
||||
python setup.py bdist_wheel
|
||||
ls -l dist
|
||||
|
||||
install: clean ## install the package to the active Python's site-packages
|
||||
python setup.py install
|
||||
37
README.rst
Normal file
37
README.rst
Normal file
@@ -0,0 +1,37 @@
|
||||
===========
|
||||
gamescrapyr
|
||||
===========
|
||||
|
||||
|
||||
.. image:: https://img.shields.io/pypi/v/gamescrapyr.svg
|
||||
:target: https://pypi.python.org/pypi/gamescrapyr
|
||||
|
||||
.. image:: https://img.shields.io/travis/anthonyscorrea/gamescrapyr.svg
|
||||
:target: https://travis-ci.com/anthonyscorrea/gamescrapyr
|
||||
|
||||
.. image:: https://readthedocs.org/projects/gamescrapyr/badge/?version=latest
|
||||
:target: https://gamescrapyr.readthedocs.io/en/latest/?version=latest
|
||||
:alt: Documentation Status
|
||||
|
||||
|
||||
|
||||
|
||||
Unofficial GameChanger scraper.
|
||||
|
||||
|
||||
* Free software: MIT license
|
||||
* Documentation: https://gamescrapyr.readthedocs.io.
|
||||
|
||||
|
||||
Features
|
||||
--------
|
||||
|
||||
* TODO
|
||||
|
||||
Credits
|
||||
-------
|
||||
|
||||
This package was created with Cookiecutter_ and the `audreyr/cookiecutter-pypackage`_ project template.
|
||||
|
||||
.. _Cookiecutter: https://github.com/audreyr/cookiecutter
|
||||
.. _`audreyr/cookiecutter-pypackage`: https://github.com/audreyr/cookiecutter-pypackage
|
||||
20
docs/Makefile
Normal file
20
docs/Makefile
Normal file
@@ -0,0 +1,20 @@
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line.
|
||||
SPHINXOPTS =
|
||||
SPHINXBUILD = python -msphinx
|
||||
SPHINXPROJ = gamescrapyr
|
||||
SOURCEDIR = .
|
||||
BUILDDIR = _build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help:
|
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile
|
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile
|
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
1
docs/authors.rst
Normal file
1
docs/authors.rst
Normal file
@@ -0,0 +1 @@
|
||||
.. include:: ../AUTHORS.rst
|
||||
162
docs/conf.py
Executable file
162
docs/conf.py
Executable file
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# gamescrapyr documentation build configuration file, created by
|
||||
# sphinx-quickstart on Fri Jun 9 13:47:02 2017.
|
||||
#
|
||||
# This file is execfile()d with the current directory set to its
|
||||
# containing dir.
|
||||
#
|
||||
# Note that not all possible configuration values are present in this
|
||||
# autogenerated file.
|
||||
#
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another
|
||||
# directory, add these directories to sys.path here. If the directory is
|
||||
# relative to the documentation root, use os.path.abspath to make it
|
||||
# absolute, like shown here.
|
||||
#
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.abspath('..'))
|
||||
|
||||
import gamescrapyr
|
||||
|
||||
# -- General configuration ---------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#
|
||||
# needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be
|
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix(es) of source filenames.
|
||||
# You can specify multiple suffix as a list of string:
|
||||
#
|
||||
# source_suffix = ['.rst', '.md']
|
||||
source_suffix = '.rst'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = 'gamescrapyr'
|
||||
copyright = "2022, Anthony Correa"
|
||||
author = "Anthony Correa"
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement
|
||||
# for |version| and |release|, also used in various other places throughout
|
||||
# the built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
version = gamescrapyr.__version__
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = gamescrapyr.__version__
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#
|
||||
# This is also used if you do content translation via gettext catalogs.
|
||||
# Usually you set "language" from the command line for these cases.
|
||||
language = None
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# This patterns also effect to html_static_path and html_extra_path
|
||||
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'sphinx'
|
||||
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = False
|
||||
|
||||
|
||||
# -- Options for HTML output -------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
#
|
||||
html_theme = 'alabaster'
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a
|
||||
# theme further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
#
|
||||
# html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ['_static']
|
||||
|
||||
|
||||
# -- Options for HTMLHelp output ---------------------------------------
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'gamescrapyrdoc'
|
||||
|
||||
|
||||
# -- Options for LaTeX output ------------------------------------------
|
||||
|
||||
latex_elements = {
|
||||
# The paper size ('letterpaper' or 'a4paper').
|
||||
#
|
||||
# 'papersize': 'letterpaper',
|
||||
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
#
|
||||
# 'pointsize': '10pt',
|
||||
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#
|
||||
# 'preamble': '',
|
||||
|
||||
# Latex figure (float) alignment
|
||||
#
|
||||
# 'figure_align': 'htbp',
|
||||
}
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, documentclass
|
||||
# [howto, manual, or own class]).
|
||||
latex_documents = [
|
||||
(master_doc, 'gamescrapyr.tex',
|
||||
'gamescrapyr Documentation',
|
||||
'Anthony Correa', 'manual'),
|
||||
]
|
||||
|
||||
|
||||
# -- Options for manual page output ------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
(master_doc, 'gamescrapyr',
|
||||
'gamescrapyr Documentation',
|
||||
[author], 1)
|
||||
]
|
||||
|
||||
|
||||
# -- Options for Texinfo output ----------------------------------------
|
||||
|
||||
# Grouping the document tree into Texinfo files. List of tuples
|
||||
# (source start file, target name, title, author,
|
||||
# dir menu entry, description, category)
|
||||
texinfo_documents = [
|
||||
(master_doc, 'gamescrapyr',
|
||||
'gamescrapyr Documentation',
|
||||
author,
|
||||
'gamescrapyr',
|
||||
'One line description of project.',
|
||||
'Miscellaneous'),
|
||||
]
|
||||
|
||||
|
||||
|
||||
1
docs/contributing.rst
Normal file
1
docs/contributing.rst
Normal file
@@ -0,0 +1 @@
|
||||
.. include:: ../CONTRIBUTING.rst
|
||||
1
docs/history.rst
Normal file
1
docs/history.rst
Normal file
@@ -0,0 +1 @@
|
||||
.. include:: ../HISTORY.rst
|
||||
20
docs/index.rst
Normal file
20
docs/index.rst
Normal file
@@ -0,0 +1,20 @@
|
||||
Welcome to gamescrapyr's documentation!
|
||||
======================================
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
:caption: Contents:
|
||||
|
||||
readme
|
||||
installation
|
||||
usage
|
||||
modules
|
||||
contributing
|
||||
authors
|
||||
history
|
||||
|
||||
Indices and tables
|
||||
==================
|
||||
* :ref:`genindex`
|
||||
* :ref:`modindex`
|
||||
* :ref:`search`
|
||||
51
docs/installation.rst
Normal file
51
docs/installation.rst
Normal file
@@ -0,0 +1,51 @@
|
||||
.. highlight:: shell
|
||||
|
||||
============
|
||||
Installation
|
||||
============
|
||||
|
||||
|
||||
Stable release
|
||||
--------------
|
||||
|
||||
To install gamescrapyr, run this command in your terminal:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ pip install gamescrapyr
|
||||
|
||||
This is the preferred method to install gamescrapyr, as it will always install the most recent stable release.
|
||||
|
||||
If you don't have `pip`_ installed, this `Python installation guide`_ can guide
|
||||
you through the process.
|
||||
|
||||
.. _pip: https://pip.pypa.io
|
||||
.. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/
|
||||
|
||||
|
||||
From sources
|
||||
------------
|
||||
|
||||
The sources for gamescrapyr can be downloaded from the `Github repo`_.
|
||||
|
||||
You can either clone the public repository:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ git clone git://github.com/anthonyscorrea/gamescrapyr
|
||||
|
||||
Or download the `tarball`_:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ curl -OJL https://github.com/anthonyscorrea/gamescrapyr/tarball/master
|
||||
|
||||
Once you have a copy of the source, you can install it with:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ python setup.py install
|
||||
|
||||
|
||||
.. _Github repo: https://github.com/anthonyscorrea/gamescrapyr
|
||||
.. _tarball: https://github.com/anthonyscorrea/gamescrapyr/tarball/master
|
||||
36
docs/make.bat
Normal file
36
docs/make.bat
Normal file
@@ -0,0 +1,36 @@
|
||||
@ECHO OFF
|
||||
|
||||
pushd %~dp0
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=python -msphinx
|
||||
)
|
||||
set SOURCEDIR=.
|
||||
set BUILDDIR=_build
|
||||
set SPHINXPROJ=gamescrapyr
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL
|
||||
if errorlevel 9009 (
|
||||
echo.
|
||||
echo.The Sphinx module was not found. Make sure you have Sphinx installed,
|
||||
echo.then set the SPHINXBUILD environment variable to point to the full
|
||||
echo.path of the 'sphinx-build' executable. Alternatively you may add the
|
||||
echo.Sphinx directory to PATH.
|
||||
echo.
|
||||
echo.If you don't have Sphinx installed, grab it from
|
||||
echo.http://sphinx-doc.org/
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
|
||||
goto end
|
||||
|
||||
:help
|
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
|
||||
|
||||
:end
|
||||
popd
|
||||
1
docs/readme.rst
Normal file
1
docs/readme.rst
Normal file
@@ -0,0 +1 @@
|
||||
.. include:: ../README.rst
|
||||
7
docs/usage.rst
Normal file
7
docs/usage.rst
Normal file
@@ -0,0 +1,7 @@
|
||||
=====
|
||||
Usage
|
||||
=====
|
||||
|
||||
To use gamescrapyr in a project::
|
||||
|
||||
import gamescrapyr
|
||||
5
gamescrapyr/__init__.py
Normal file
5
gamescrapyr/__init__.py
Normal file
@@ -0,0 +1,5 @@
|
||||
"""Top-level package for gamescrapyr."""
|
||||
|
||||
__author__ = """Anthony Correa"""
|
||||
__email__ = 'a@correa.co'
|
||||
__version__ = '0.1.0'
|
||||
225
gamescrapyr/gamescrapyr.py
Normal file
225
gamescrapyr/gamescrapyr.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""Main module."""
|
||||
from os import getenv
|
||||
|
||||
import requests
|
||||
import json
|
||||
import re
|
||||
import csv
|
||||
from bs4 import BeautifulSoup
|
||||
import datetime
|
||||
import pytz
|
||||
|
||||
|
||||
class GameChangerClient():
|
||||
session = requests.Session()
|
||||
url = "https://gc.com/t/{season_slug}/{team_slug}-{team_id}/{page}"
|
||||
|
||||
def __init__(self, email, password):
|
||||
r = requests.get("https://gc.com/login")
|
||||
|
||||
payload = {
|
||||
'email': email,
|
||||
'password': password,
|
||||
"csrfmiddlewaretoken": r.cookies.get("csrftoken")
|
||||
}
|
||||
|
||||
self.session.headers["referer"] = "https://gc.com/do-login"
|
||||
|
||||
signed_in = self.session.post(
|
||||
url="https://gc.com/do-login",
|
||||
headers=self.session.headers,
|
||||
cookies=r.cookies,
|
||||
data=payload
|
||||
)
|
||||
|
||||
if signed_in.status_code != 200:
|
||||
raise Exception(f'Login not successful, return code of {signed_in.status_code} for {signed_in.reason}')
|
||||
elif b"Sign In to GC Classic" in signed_in.content or b"Sign Out" not in signed_in.content:
|
||||
raise Exception (f'Login not successful, returned content not as expected')
|
||||
|
||||
self.session.headers.update(
|
||||
{
|
||||
"x-csrftoken": r.cookies.get("csrftoken"),
|
||||
}
|
||||
)
|
||||
|
||||
pass
|
||||
|
||||
def _scrape_from_page_initialize_json(self, season_slug, team_id, team_slug, page):
|
||||
resp = self.session.get(
|
||||
self.url.format(season_slug=season_slug, team_id=team_id, team_slug=team_slug, page=page)
|
||||
)
|
||||
initialize_page_json = re.search(
|
||||
r'page.initialize\(\$.parseJSON\("(.*?)"\)', resp.content.decode("unicode_escape")
|
||||
)
|
||||
m = initialize_page_json.group(1)
|
||||
return json.loads(m)
|
||||
|
||||
def get_roster(self, season_slug, team_id, team_slug):
|
||||
data = self._scrape_from_page_initialize_json(season_slug, team_id, team_slug, "roster")
|
||||
return data['roster']
|
||||
|
||||
def get_stats(self, season_slug, team_id, team_slug):
|
||||
resp = self.session.get(
|
||||
self.url.format(
|
||||
page="stats/batting/Qualified/standard/csv",
|
||||
season_slug = season_slug, team_id = team_id, team_slug = team_slug
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
resp.status_code != 200
|
||||
or b"Please sign in or join to continue." in resp.content
|
||||
):
|
||||
raise Exception("Stats fetch failed.")
|
||||
|
||||
roster = self.get_roster(season_slug = season_slug, team_id = team_id, team_slug = team_slug)
|
||||
|
||||
id_lookup = {
|
||||
(p.get("fname"), p.get("lname")): p.get("player_id")
|
||||
for p in roster
|
||||
}
|
||||
|
||||
decoded_content = resp.content.decode("utf-8")
|
||||
|
||||
cr = csv.reader(decoded_content.splitlines(), delimiter=",")
|
||||
my_list = list(cr)
|
||||
player_keys = [
|
||||
(i, key)
|
||||
for i, key in enumerate(my_list[1][: my_list[0].index("Offensive Stats")])
|
||||
]
|
||||
offensive_keys = [
|
||||
(i, key)
|
||||
for i, key in enumerate(
|
||||
my_list[1][
|
||||
my_list[0]
|
||||
.index("Offensive Stats"): my_list[0]
|
||||
.index("Defensive Stats")
|
||||
- 1
|
||||
],
|
||||
start=my_list[0].index("Offensive Stats"),
|
||||
)
|
||||
]
|
||||
defensive_keys = [
|
||||
(i, key)
|
||||
for i, key in enumerate(
|
||||
my_list[1][my_list[0].index("Defensive Stats"):],
|
||||
start=my_list[0].index("Defensive Stats"),
|
||||
)
|
||||
]
|
||||
|
||||
stats = {}
|
||||
for row in my_list[2:]:
|
||||
player_keys
|
||||
number, lname, fname = row[:3]
|
||||
if number == "Team":
|
||||
break
|
||||
gamechanger_id = id_lookup[(fname, lname)]
|
||||
stats[gamechanger_id] = {
|
||||
"offensive": {k: row[i] for i, k in offensive_keys},
|
||||
"defensive": {k: row[i] for i, k in defensive_keys},
|
||||
}
|
||||
|
||||
return stats
|
||||
|
||||
def get_games(self, season_slug, team_id, team_slug):
|
||||
page = self.session.get(
|
||||
url=self.url.format(
|
||||
page="schedule/games",
|
||||
team_id=team_id, season_slug =season_slug, team_slug=team_slug),
|
||||
)
|
||||
soup = BeautifulSoup(page.content, "html.parser")
|
||||
game_elements = [r for r in soup.find_all("tr") if "game" in r.attrs.get("class")]
|
||||
games = []
|
||||
for i, game_element in enumerate(game_elements):
|
||||
game_slug = game_element.find("a").attrs.get("href").split("/")[1]
|
||||
title = game_element.find("a").text
|
||||
jslocaldate, jslocaltime_start, jslocaltime_arrival = (
|
||||
t.attrs.get("datetime") for t in game_element.findAll("time")
|
||||
)
|
||||
games.append(
|
||||
{
|
||||
"id": game_element.attrs.get("data-id"),
|
||||
"title": title,
|
||||
"game_slug": game_slug,
|
||||
"start": pytz.timezone("utc").localize(
|
||||
datetime.datetime.fromisoformat(jslocaltime_start)
|
||||
),
|
||||
}
|
||||
)
|
||||
return games
|
||||
|
||||
def submit_lineup(self, lineup, season_slug, team_slug, team_id):
|
||||
headers = self.session.headers
|
||||
headers.update(
|
||||
{
|
||||
"referer": self.url.format(
|
||||
season_slug=season_slug,
|
||||
team_slug=team_slug,
|
||||
team_id=team_id,
|
||||
page="lineup_edit",
|
||||
),
|
||||
"x-csrftoken": self.session.cookies.get("csrftoken"),
|
||||
"Content-Type": "application/x-www-form-urlencoded;",
|
||||
}
|
||||
)
|
||||
resp = self.session.post(
|
||||
headers = headers,
|
||||
cookies= self.session.cookies,
|
||||
url= f"https://gc.com/do-save-lineup/{team_id}",
|
||||
json={"lineup": lineup},
|
||||
)
|
||||
|
||||
if resp.status_code == 200 and resp.content == b"OK":
|
||||
return resp
|
||||
else:
|
||||
raise requests.exceptions.RequestException(
|
||||
f"Returned {resp.status_code} for {resp.reason}"
|
||||
)
|
||||
|
||||
def get_game_stream(self, game_slug):
|
||||
game_page = self.session.get(
|
||||
cookies=self.session.cookies,
|
||||
url=f"https://gc.com/{game_slug}",
|
||||
)
|
||||
game_soup = BeautifulSoup(game_page.content, "html.parser")
|
||||
data_push_url_rel = game_soup.find("body").attrs.get("data-push-url")[2:]
|
||||
data_push_url = f"https://{data_push_url_rel}?sabertooth_aware=true"
|
||||
stream_page = self.session.get(
|
||||
cookies=self.session.cookies, url=data_push_url
|
||||
)
|
||||
game_stream = json.loads(stream_page.content)
|
||||
return game_stream
|
||||
|
||||
def get_lineup(self, season_slug, team_slug, team_id):
|
||||
resp = self.session.get(
|
||||
self.url.format(
|
||||
page="lineup",
|
||||
season_slug=season_slug, team_id=team_id, team_slug=team_slug)
|
||||
)
|
||||
|
||||
soup = BeautifulSoup(resp.content, "html.parser")
|
||||
lineup_table = soup.find('table', {'id':'tbl_lineup'})
|
||||
lineup_rows = [r for r in lineup_table.findAll('tr')[1:] if r.text !='\nHas a Designated Hitter:\nPosition\n']
|
||||
lineup = []
|
||||
for row in lineup_rows:
|
||||
order_element, player_element, position_element = row.findAll('td')
|
||||
player_href = player_element.find(href=True)['href']
|
||||
player_id = player_href.split('-')[1]
|
||||
order = order_element.text
|
||||
d = {
|
||||
'player_id': player_id,
|
||||
'position': position_element.text
|
||||
}
|
||||
if order:
|
||||
lineup.append(d)
|
||||
elif not order:
|
||||
dhs = [(i, r) for i, r in enumerate(lineup) if r['position']=='DH']
|
||||
if not dhs:
|
||||
raise Exception("No DHs")
|
||||
else:
|
||||
dh_index, dh = dhs[0]
|
||||
dh['forwhom'] = player_id
|
||||
lineup.insert(dh_index+1, d)
|
||||
|
||||
return lineup
|
||||
12
requirements_dev.txt
Normal file
12
requirements_dev.txt
Normal file
@@ -0,0 +1,12 @@
|
||||
pip==19.2.3
|
||||
bump2version==0.5.11
|
||||
wheel==0.33.6
|
||||
watchdog==0.9.0
|
||||
flake8==3.7.8
|
||||
tox==3.14.0
|
||||
coverage==4.5.4
|
||||
Sphinx==1.8.5
|
||||
twine==1.14.0
|
||||
|
||||
|
||||
black==21.7b0
|
||||
18
setup.cfg
Normal file
18
setup.cfg
Normal file
@@ -0,0 +1,18 @@
|
||||
[bumpversion]
|
||||
current_version = 0.1.0
|
||||
commit = True
|
||||
tag = True
|
||||
|
||||
[bumpversion:file:setup.py]
|
||||
search = version='{current_version}'
|
||||
replace = version='{new_version}'
|
||||
|
||||
[bumpversion:file:gamescrapyr/__init__.py]
|
||||
search = __version__ = '{current_version}'
|
||||
replace = __version__ = '{new_version}'
|
||||
|
||||
[bdist_wheel]
|
||||
universal = 1
|
||||
|
||||
[flake8]
|
||||
exclude = docs
|
||||
44
setup.py
Normal file
44
setup.py
Normal file
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
"""The setup script."""
|
||||
|
||||
from setuptools import setup, find_packages
|
||||
|
||||
with open('README.rst') as readme_file:
|
||||
readme = readme_file.read()
|
||||
|
||||
with open('HISTORY.rst') as history_file:
|
||||
history = history_file.read()
|
||||
|
||||
requirements = [ ]
|
||||
|
||||
test_requirements = [ ]
|
||||
|
||||
setup(
|
||||
author="Anthony Correa",
|
||||
author_email='a@correa.co',
|
||||
python_requires='>=3.6',
|
||||
classifiers=[
|
||||
'Development Status :: 2 - Pre-Alpha',
|
||||
'Intended Audience :: Developers',
|
||||
'License :: OSI Approved :: MIT License',
|
||||
'Natural Language :: English',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
],
|
||||
description="Unofficial GameChanger scraper.",
|
||||
install_requires=requirements,
|
||||
license="MIT license",
|
||||
long_description=readme + '\n\n' + history,
|
||||
include_package_data=True,
|
||||
keywords='gamescrapyr',
|
||||
name='gamescrapyr',
|
||||
packages=find_packages(include=['gamescrapyr', 'gamescrapyr.*']),
|
||||
test_suite='tests',
|
||||
tests_require=test_requirements,
|
||||
url='https://github.com/anthonyscorrea/gamescrapyr',
|
||||
version='0.1.0',
|
||||
zip_safe=False,
|
||||
)
|
||||
19
tox.ini
Normal file
19
tox.ini
Normal file
@@ -0,0 +1,19 @@
|
||||
[tox]
|
||||
envlist = py36, py37, py38, flake8
|
||||
|
||||
[travis]
|
||||
python =
|
||||
3.8: py38
|
||||
3.7: py37
|
||||
3.6: py36
|
||||
|
||||
[testenv:flake8]
|
||||
basepython = python
|
||||
deps = flake8
|
||||
commands = flake8 gamescrapyr tests
|
||||
|
||||
[testenv]
|
||||
setenv =
|
||||
PYTHONPATH = {toxinidir}
|
||||
|
||||
commands = python setup.py test
|
||||
Reference in New Issue
Block a user