diff --git a/.dockerignore b/.dockerignore index 30a9ca21..46267e90 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ -.travis.yml .git .gitignore docker-compose.yml diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml new file mode 100644 index 00000000..8a469ef6 --- /dev/null +++ b/.github/workflows/tests-ubuntu.yml @@ -0,0 +1,38 @@ +name: Ubuntu Tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + schedule: + - cron: '0 9 * * 4' + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: install dependencies + run: pip install --upgrade -r requirements.txt + - name: fetch upstream cheat sheets + run: python lib/fetch.py fetch-all + - name: run bash tests + run: bash tests/run-tests.sh + - name: run pytest + run: pytest lib/ + + docker: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - run: docker-compose build + - run: docker images + - run: | + docker-compose -f docker-compose.yml up -d + # docker-compose -f docker-compose.yml -f docker-compose.debug.yml up -d + docker-compose ps + # wait until the web server is up + wget --timeout 3 --tries=5 --spider localhost:8002 2>&1 | grep -i http + docker-compose logs --no-color + - run: CHEATSH_TEST_STANDALONE=NO bash tests/run-tests.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f64a78ee..00000000 --- a/.travis.yml +++ /dev/null @@ -1,17 +0,0 @@ -dist: bionic - -language: - - generic - -before_install: - - docker-compose build - - docker images - - docker-compose -f docker-compose.yml -f docker-compose.debug.yml up -d - - docker-compose ps - -script: - - sleep 3 - - curl http://localhost:8002 - - docker-compose logs --no-color - - docker logs chtsh - - CHEATSH_TEST_STANDALONE=NO bash tests/run-tests.sh diff --git a/Dockerfile b/Dockerfile index e41aa0a5..dba6b242 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,17 @@ -FROM alpine:3.12 +FROM alpine:3.14 # fetching cheat sheets ## installing dependencies RUN apk add --update --no-cache git py3-six py3-pygments py3-yaml py3-gevent \ - libstdc++ py3-colorama py3-requests py3-icu py3-redis -## building missing python packages -RUN apk add --no-cache --virtual build-deps py3-pip g++ python3-dev \ - && pip3 install --no-cache-dir rapidfuzz colored polyglot pycld2 \ - && apk del build-deps + libstdc++ py3-colorama py3-requests py3-icu py3-redis sed ## copying WORKDIR /app COPY . /app +## building missing python packages +RUN apk add --no-cache --virtual build-deps py3-pip g++ python3-dev libffi-dev \ + && pip3 install --no-cache-dir --upgrade pygments \ + && pip3 install --no-cache-dir -r requirements.txt \ + && apk del build-deps +# fetching dependencies RUN mkdir -p /root/.cheat.sh/log/ \ && python3 lib/fetch.py fetch-all diff --git a/README.md b/README.md index b0f861ac..4af6d786 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ + ![cheat.sh logo](http://cheat.sh/files/big-logo-v2-fixed.png) Unified access to the best community driven cheat sheets repositories of the world. @@ -15,22 +16,21 @@ What features should it have? * **Tutoring** — It should help you to learn the subject. * **Inconspicuous** — It should be possible to use it completely unnoticed. -Such a thing exists. +Such a thing exists! It's easy to [install](#installation) and there's even [auto-complete](#tab-completion). -[![Build Status](https://travis-ci.org/chubin/cheat.sh.svg?branch=master)](https://travis-ci.org/chubin/cheat.sh) ## Features **cheat.sh** -* Has a simple curl/browser interface. +* Has a simple curl/browser/editor interface. * Covers 56 programming languages, several DBMSes, and more than 1000 most important UNIX/Linux commands. * Provides access to the best community driven cheat sheets repositories in the world, on par with StackOverflow. -* Available everywhere, no installation needed. +* Available everywhere, no installation needed, but can be installed for offline usage. * Ultrafast, returns answers within 100 ms, as a rule. * Has a convenient command line client, `cht.sh`, that is very advantageous and helpful, though not mandatory. * Can be used directly from code editors, without opening a browser and not switching your mental context. -* Supports a special stealth mode where it can be used fully invisibly without ever touching a key and and making sounds. +* Supports a special stealth mode where it can be used fully invisibly without ever touching a key and making sounds.

@@ -44,7 +44,10 @@ Such a thing exists. * [Installation](#installation) * [Client usage](#client-usage) * [Tab-completion](#tab-completion) + - [Bash Tab completion](#bash-tab-completion) + - [ZSH Tab completion](#zsh-tab-completion) * [Stealth mode](#stealth-mode) + * [Windows command line client](#windows-command-line-client) * [Self-Hosting](#self-hosting) * [Docker](#docker) * [Editors integration](#editors-integration) @@ -188,6 +191,8 @@ Try your own queries. Follow these rules: Read more about the programming languages queries below. +---- + ## Command line client, cht.sh The cheat.sh service has its own command line client (`cht.sh`) that @@ -203,17 +208,18 @@ has several useful features compared to querying the service directly with `curl To install the client: -``` - mkdir -p ~/bin/ - curl https://cht.sh/:cht.sh > ~/bin/cht.sh - chmod +x ~/bin/cht.sh +```bash +PATH_DIR="$HOME/bin" # or another directory on your $PATH +mkdir -p "$PATH_DIR" +curl https://cht.sh/:cht.sh > "$PATH_DIR/cht.sh" +chmod +x "$PATH_DIR/cht.sh" ``` or to install it globally (for all users): -``` - curl https://cht.sh/:cht.sh | sudo tee /usr/local/bin/cht.sh - chmod +x /usr/local/bin/cht.sh +```bash +curl https://cht.sh/:cht.sh | sudo tee /usr/local/bin/cht.sh +chmod +x /usr/local/bin/cht.sh ``` Note: The package "rlwrap" is a required dependency to run in shell mode. Install this using `sudo apt install rlwrap` @@ -301,13 +307,13 @@ Use it to specify query options that you would use with each query. For example, to switch syntax highlighting off create the file with the following content: -``` +```bash CHTSH_QUERY_OPTIONS="T" ``` Or if you want to use a special syntax highlighting theme: -``` +```bash CHTSH_QUERY_OPTIONS="style=native" ``` @@ -315,7 +321,7 @@ CHTSH_QUERY_OPTIONS="style=native" Other cht.sh configuration parameters: -``` +```bash CHTSH_CURL_OPTIONS="-A curl" # curl options used for cht.sh queries CHTSH_URL=https://cht.sh # URL of the cheat.sh server ``` @@ -327,22 +333,24 @@ CHTSH_URL=https://cht.sh # URL of the cheat.sh server To activate tab completion support for `cht.sh`, add the `:bash_completion` script to your `~/.bashrc`: -``` - $ curl https://cheat.sh/:bash_completion > ~/.bash.d/cht.sh - $ . ~/.bash.d/cht.sh - $ # and add . ~/.bash.d/cht.sh to ~/.bashrc +```bash + curl https://cheat.sh/:bash_completion > ~/.bash.d/cht.sh + . ~/.bash.d/cht.sh + # and add . ~/.bash.d/cht.sh to ~/.bashrc ``` #### ZSH Tab completion To activate tab completion support for `cht.sh`, add the `:zsh` script to the *fpath* in your `~/.zshrc`: -``` - $ curl https://cheat.sh/:zsh > ~/.zsh.d/_cht - $ echo 'fpath=(~/.zsh.d/ $fpath)' >> ~/.zshrc - $ # Open a new shell to load the plugin +```zsh + curl https://cheat.sh/:zsh > ~/.zsh.d/_cht + echo 'fpath=(~/.zsh.d/ $fpath)' >> ~/.zshrc + # Open a new shell to load the plugin ``` +---- + ### Stealth mode Being used fully unnoticed is one of the most important property of any cheat sheet. @@ -443,6 +451,8 @@ You can also use [`scoop`](https://github.com/lukesampson/scoop) command-line in scoop install cht ``` +---- + ## Self-Hosting ### Docker @@ -530,7 +540,7 @@ In this example, several Vim plugins are used: * [scrooloose/syntastic](https://github.com/vim-syntastic/syntastic) — Syntax checking plugin * [cheat.sh-vim](https://github.com/dbeniamine/cheat.sh-vim) — Vim support -Syntastic shows warnings and errors (found by code analysis tools: `jshint`, `merlin`, `pylint`, `shellcheckt etc.), +Syntastic shows warnings and errors (found by code analysis tools: `jshint`, `merlin`, `pylint`, `shellcheck` etc.), and `cheat.sh-vim` shows you explanations for the errors and warnings and answers on programming languages queries written in the editor. @@ -658,6 +668,7 @@ Other pages: :post how to post new cheat sheet :styles list of color styles :styles-demo show color styles usage examples + :random fetches a random page (can be used in a subsection too: /go/:random) ``` ## Search @@ -813,15 +824,17 @@ and information sources, maintained by thousands of users, developers and author all over the world (in the *Users* column number of contributors/number of stars is shown): -|Cheat sheets |Repository | Users | Creation Date | -|-----------------------|------------------------------------------------------|------------|---------------| -|UNIX/Linux, programming|[cheat.sheets](https://github.com/chubin/cheat.sheets)| 38/223 | May 1, 2017 | -|UNIX/Linux commands |[tldr-pages/tldr](https://github.com/tldr-pages/tldr) | 760/23158 | Dec 8, 2013 | -|UNIX/Linux commands |[chrisallenlane/cheat](https://github.com/chrisallenlane/cheat)|131/5240|Jul 28, 2013| -|Programming languages |[adambard/learnxinyminutes-docs](https://github.com/adambard/learnxinyminutes-docs)|1246/6748|Jun 23, 2013| -|Go |[a8m/go-lang-cheat-sheet](https://github.com/a8m/go-lang-cheat-sheet)|31/4039|Feb 9, 2014| -|Perl |[pkrumnis/perl1line.txt](https://github.com/pkrumins/perl1line.txt)|5/190|Nov 4, 2011| -|Programming languages |[StackOverflow](https://stackoverflow.com)|9M |Sep 15, 2008| +|Cheat sheets |Repository |C/U* |Stars |Creation Date| +|-----------------------|------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------|-------------| +|UNIX/Linux, programming|[cheat.sheets](https://github.com/chubin/cheat.sheets) |![](https://img.shields.io/github/contributors-anon/chubin/cheat.sheets?label=%F0%9F%91%A5&labelColor=white) |![](https://img.shields.io/github/stars/chubin/cheat.sheets?label=%E2%AD%90&labelColor=white) |May 1, 2017 | +|UNIX/Linux commands |[tldr-pages/tldr](https://github.com/tldr-pages/tldr) |![](https://img.shields.io/github/contributors-anon/tldr-pages/tldr?label=%F0%9F%91%A5&labelColor=white) |![](https://img.shields.io/github/stars/tldr-pages/tldr?label=%E2%AD%90&labelColor=white) |Dec 8, 2013 | +|UNIX/Linux commands |[chrisallenlane/cheat](https://github.com/chrisallenlane/cheat) |![](https://img.shields.io/github/contributors-anon/chrisallenlane/cheat?label=%F0%9F%91%A5&labelColor=white) |![](https://img.shields.io/github/stars/chrisallenlane/cheat?label=%E2%AD%90&labelColor=white) |Jul 28, 2013 | +|Programming languages |[adambard/learnxinyminutes-docs](https://github.com/adambard/learnxinyminutes-docs) |![](https://img.shields.io/github/contributors-anon/adambard/learnxinyminutes-docs?label=%F0%9F%91%A5&labelColor=white)|![](https://img.shields.io/github/stars/adambard/learnxinyminutes-docs?label=%E2%AD%90&labelColor=white)|Jun 23, 2013 | +|Go |[a8m/go-lang-cheat-sheet](https://github.com/a8m/go-lang-cheat-sheet) |![](https://img.shields.io/github/contributors-anon/a8m/go-lang-cheat-sheet?label=%F0%9F%91%A5&labelColor=white) |![](https://img.shields.io/github/stars/a8m/go-lang-cheat-sheet?label=%E2%AD%90&labelColor=white) |Feb 9, 2014 | +|Perl |[pkrumnis/perl1line.txt](https://github.com/pkrumins/perl1line.txt) |![](https://img.shields.io/github/contributors-anon/pkrumins/perl1line.txt?label=%F0%9F%91%A5&labelColor=white) |![](https://img.shields.io/github/stars/pkrumins/perl1line.txt?label=%E2%AD%90&labelColor=white) |Nov 4, 2011 | +|Programming languages |[StackOverflow](https://stackoverflow.com) |[14M](https://stackexchange.com/leagues/1/alltime/stackoverflow) |N/A |Sep 15, 2008 | + +(*) C/U — contributors for GitHub repositories, Users for Stackoverflow Pie diagram reflecting cheat sheets sources distribution (by number of cheat sheets on cheat.sh originating from a repository): @@ -865,3 +878,15 @@ If you want to add a cheat sheet repository to cheat.sh, please open an issue: * [Add a new repository](https://github.com/chubin/cheat.sh/issues/new) Please specify the name of the repository, and give its short description. + + +## Installation and standalone usage + +You don't need to install anything, to start using *cheat.sh*. + +There are two cases, when you want to install *cheat.sh* locally: + +1. You plan to use it off-line, without Internet access; +2. You want to use your own cheat sheets (additionally, or as a replacement). + +Installation process in described in details here: [cheat.sh standalone installation](doc/standalone.md) diff --git a/bin/app.py b/bin/app.py new file mode 100644 index 00000000..d965bd7b --- /dev/null +++ b/bin/app.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python +# vim: set encoding=utf-8 +# pylint: disable=wrong-import-position,wrong-import-order + +""" +Main server program. + +Configuration parameters: + + path.internal.malformed + path.internal.static + path.internal.templates + path.log.main + path.log.queries +""" + +from __future__ import print_function + +import sys +if sys.version_info[0] < 3: + reload(sys) + sys.setdefaultencoding('utf8') + +import sys +import logging +import os +import requests +import jinja2 +from flask import Flask, request, send_from_directory, redirect, Response + +sys.path.append(os.path.abspath(os.path.join(__file__, "..", "..", "lib"))) +from config import CONFIG +from limits import Limits +from cheat_wrapper import cheat_wrapper +from post import process_post_request +from options import parse_args + +from stateful_queries import save_query, last_query + + +if not os.path.exists(os.path.dirname(CONFIG["path.log.main"])): + os.makedirs(os.path.dirname(CONFIG["path.log.main"])) +logging.basicConfig( + filename=CONFIG["path.log.main"], + level=logging.DEBUG, + format='%(asctime)s %(message)s') +# Fix Flask "exception and request logging" to `stderr`. +# +# When Flask's werkzeug detects that logging is already set, it +# doesn't add its own logger that prints exceptions. +stderr_handler = logging.StreamHandler() +logging.getLogger().addHandler(stderr_handler) +# +# Alter log format to disting log lines from everything else +stderr_handler.setFormatter(logging.Formatter('%(filename)s:%(lineno)s: %(message)s')) +# +# Sometimes werkzeug starts logging before an app is imported +# (https://github.com/pallets/werkzeug/issues/1969) +# resulting in duplicating lines. In that case we need root +# stderr handler to skip lines from werkzeug. +class SkipFlaskLogger(object): + def filter(self, record): + if record.name != 'werkzeug': + return True +if logging.getLogger('werkzeug').handlers: + stderr_handler.addFilter(SkipFlaskLogger()) + + +app = Flask(__name__) # pylint: disable=invalid-name +app.jinja_loader = jinja2.ChoiceLoader([ + app.jinja_loader, + jinja2.FileSystemLoader(CONFIG["path.internal.templates"])]) + +LIMITS = Limits() + +PLAIN_TEXT_AGENTS = [ + "curl", + "httpie", + "lwp-request", + "wget", + "python-requests", + "openbsd ftp", + "powershell", + "fetch", + "aiohttp", +] + +def _is_html_needed(user_agent): + """ + Basing on `user_agent`, return whether it needs HTML or ANSI + """ + return all([x not in user_agent for x in PLAIN_TEXT_AGENTS]) + +def is_result_a_script(query): + return query in [':cht.sh'] + +@app.route('/files/') +def send_static(path): + """ + Return static file `path`. + Can be served by the HTTP frontend. + """ + return send_from_directory(CONFIG["path.internal.static"], path) + +@app.route('/favicon.ico') +def send_favicon(): + """ + Return static file `favicon.ico`. + Can be served by the HTTP frontend. + """ + return send_from_directory(CONFIG["path.internal.static"], 'favicon.ico') + +@app.route('/malformed-response.html') +def send_malformed(): + """ + Return static file `malformed-response.html`. + Can be served by the HTTP frontend. + """ + dirname, filename = os.path.split(CONFIG["path.internal.malformed"]) + return send_from_directory(dirname, filename) + +def log_query(ip_addr, found, topic, user_agent): + """ + Log processed query and some internal data + """ + log_entry = "%s %s %s %s\n" % (ip_addr, found, topic, user_agent) + with open(CONFIG["path.log.queries"], 'ab') as my_file: + my_file.write(log_entry.encode('utf-8')) + +def get_request_ip(req): + """ + Extract IP address from `request` + """ + + if req.headers.getlist("X-Forwarded-For"): + ip_addr = req.headers.getlist("X-Forwarded-For")[0] + if ip_addr.startswith('::ffff:'): + ip_addr = ip_addr[7:] + else: + ip_addr = req.remote_addr + if req.headers.getlist("X-Forwarded-For"): + ip_addr = req.headers.getlist("X-Forwarded-For")[0] + if ip_addr.startswith('::ffff:'): + ip_addr = ip_addr[7:] + else: + ip_addr = req.remote_addr + + return ip_addr + +def get_answer_language(request): + """ + Return preferred answer language based on + domain name, query arguments and headers + """ + + def _parse_accept_language(accept_language): + languages = accept_language.split(",") + locale_q_pairs = [] + + for language in languages: + try: + if language.split(";")[0] == language: + # no q => q = 1 + locale_q_pairs.append((language.strip(), "1")) + else: + locale = language.split(";")[0].strip() + weight = language.split(";")[1].split("=")[1] + locale_q_pairs.append((locale, weight)) + except IndexError: + pass + + return locale_q_pairs + + def _find_supported_language(accepted_languages): + for lang_tuple in accepted_languages: + lang = lang_tuple[0] + if '-' in lang: + lang = lang.split('-', 1)[0] + return lang + return None + + lang = None + hostname = request.headers['Host'] + if hostname.endswith('.cheat.sh'): + lang = hostname[:-9] + + if 'lang' in request.args: + lang = request.args.get('lang') + + header_accept_language = request.headers.get('Accept-Language', '') + if lang is None and header_accept_language: + lang = _find_supported_language( + _parse_accept_language(header_accept_language)) + + return lang + +def _proxy(*args, **kwargs): + # print "method=", request.method, + # print "url=", request.url.replace('/:shell-x/', ':3000/') + # print "headers=", {key: value for (key, value) in request.headers if key != 'Host'} + # print "data=", request.get_data() + # print "cookies=", request.cookies + # print "allow_redirects=", False + + url_before, url_after = request.url.split('/:shell-x/', 1) + url = url_before + ':3000/' + + if 'q' in request.args: + url_after = '?' + "&".join("arg=%s" % x for x in request.args['q'].split()) + + url += url_after + print(url) + print(request.get_data()) + resp = requests.request( + method=request.method, + url=url, + headers={key: value for (key, value) in request.headers if key != 'Host'}, + data=request.get_data(), + cookies=request.cookies, + allow_redirects=False) + + excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] + headers = [(name, value) for (name, value) in resp.raw.headers.items() + if name.lower() not in excluded_headers] + + response = Response(resp.content, resp.status_code, headers) + return response + + +@app.route("/", methods=['GET', 'POST']) +@app.route("/", methods=["GET", "POST"]) +def answer(topic=None): + """ + Main rendering function, it processes incoming weather queries. + Depending on user agent it returns output in HTML or ANSI format. + + Incoming data: + request.args + request.headers + request.remote_addr + request.referrer + request.query_string + """ + + user_agent = request.headers.get('User-Agent', '').lower() + html_needed = _is_html_needed(user_agent) + options = parse_args(request.args) + + if topic in ['apple-touch-icon-precomposed.png', 'apple-touch-icon.png', 'apple-touch-icon-120x120-precomposed.png'] \ + or (topic is not None and any(topic.endswith('/'+x) for x in ['favicon.ico'])): + return '' + + request_id = request.cookies.get('id') + if topic is not None and topic.lstrip('/') == ':last': + if request_id: + topic = last_query(request_id) + else: + return "ERROR: you have to set id for your requests to use /:last\n" + else: + if request_id: + save_query(request_id, topic) + + if request.method == 'POST': + process_post_request(request, html_needed) + if html_needed: + return redirect("/") + return "OK\n" + + if 'topic' in request.args: + return redirect("/%s" % request.args.get('topic')) + + if topic is None: + topic = ":firstpage" + + if topic.startswith(':shell-x/'): + return _proxy() + #return requests.get('http://127.0.0.1:3000'+topic[8:]).text + + lang = get_answer_language(request) + if lang: + options['lang'] = lang + + ip_address = get_request_ip(request) + if '+' in topic: + not_allowed = LIMITS.check_ip(ip_address) + if not_allowed: + return "429 %s\n" % not_allowed, 429 + + html_is_needed = _is_html_needed(user_agent) and not is_result_a_script(topic) + if html_is_needed: + output_format='html' + else: + output_format='ansi' + result, found = cheat_wrapper(topic, request_options=options, output_format=output_format) + if 'Please come back in several hours' in result and html_is_needed: + malformed_response = open(os.path.join(CONFIG["path.internal.malformed"])).read() + return malformed_response + + log_query(ip_address, found, topic, user_agent) + if html_is_needed: + return result + return Response(result, mimetype='text/plain') diff --git a/bin/srv.py b/bin/srv.py index 52aaff11..847375a7 100644 --- a/bin/srv.py +++ b/bin/srv.py @@ -1,286 +1,28 @@ #!/usr/bin/env python -# vim: set encoding=utf-8 -# pylint: disable=wrong-import-position,wrong-import-order - -""" -Main server program. - -Configuration parameters: - - path.internal.malformed - path.internal.static - path.internal.templates - path.log.main - path.log.queries -""" - -from __future__ import print_function - -import sys -if sys.version_info[0] < 3: - reload(sys) - sys.setdefaultencoding('utf8') - +# +# Serving cheat.sh with `gevent` +# from gevent.monkey import patch_all from gevent.pywsgi import WSGIServer patch_all() -import sys -import logging import os -import requests -import jinja2 -from flask import Flask, request, send_from_directory, redirect, Response - -sys.path.append(os.path.abspath(os.path.join(__file__, "..", "..", "lib"))) -from config import CONFIG -from limits import Limits -from cheat_wrapper import cheat_wrapper -from post import process_post_request -from options import parse_args - -from stateful_queries import save_query, last_query - -if not os.path.exists(os.path.dirname(CONFIG["path.log.main"])): - os.makedirs(os.path.dirname(CONFIG["path.log.main"])) -logging.basicConfig( - filename=CONFIG["path.log.main"], - level=logging.DEBUG, - format='%(asctime)s %(message)s') - -app = Flask(__name__) # pylint: disable=invalid-name -app.jinja_loader = jinja2.ChoiceLoader([ - app.jinja_loader, - jinja2.FileSystemLoader(CONFIG["path.internal.templates"])]) - -LIMITS = Limits() - -def is_html_needed(user_agent): - """ - Basing on `user_agent`, return whether it needs HTML or ANSI - """ - plaintext_clients = [ - 'curl', 'wget', 'fetch', 'httpie', 'lwp-request', 'openbsd ftp', 'python-requests'] - return all([x not in user_agent for x in plaintext_clients]) - -def is_result_a_script(query): - return query in [':cht.sh'] - -@app.route('/files/') -def send_static(path): - """ - Return static file `path`. - Can be served by the HTTP frontend. - """ - return send_from_directory(CONFIG["path.internal.static"], path) - -@app.route('/favicon.ico') -def send_favicon(): - """ - Return static file `favicon.ico`. - Can be served by the HTTP frontend. - """ - return send_from_directory(CONFIG["path.internal.static"], 'favicon.ico') - -@app.route('/malformed-response.html') -def send_malformed(): - """ - Return static file `malformed-response.html`. - Can be served by the HTTP frontend. - """ - dirname, filename = os.path.split(CONFIG["path.internal.malformed"]) - return send_from_directory(dirname, filename) - -def log_query(ip_addr, found, topic, user_agent): - """ - Log processed query and some internal data - """ - log_entry = "%s %s %s %s\n" % (ip_addr, found, topic, user_agent) - with open(CONFIG["path.log.queries"], 'ab') as my_file: - my_file.write(log_entry.encode('utf-8')) - -def get_request_ip(req): - """ - Extract IP address from `request` - """ - - if req.headers.getlist("X-Forwarded-For"): - ip_addr = req.headers.getlist("X-Forwarded-For")[0] - if ip_addr.startswith('::ffff:'): - ip_addr = ip_addr[7:] - else: - ip_addr = req.remote_addr - if req.headers.getlist("X-Forwarded-For"): - ip_addr = req.headers.getlist("X-Forwarded-For")[0] - if ip_addr.startswith('::ffff:'): - ip_addr = ip_addr[7:] - else: - ip_addr = req.remote_addr - - return ip_addr - -def get_answer_language(request): - """ - Return preferred answer language based on - domain name, query arguments and headers - """ - - def _parse_accept_language(accept_language): - languages = accept_language.split(",") - locale_q_pairs = [] - - for language in languages: - try: - if language.split(";")[0] == language: - # no q => q = 1 - locale_q_pairs.append((language.strip(), "1")) - else: - locale = language.split(";")[0].strip() - weight = language.split(";")[1].split("=")[1] - locale_q_pairs.append((locale, weight)) - except IndexError: - pass - - return locale_q_pairs - - def _find_supported_language(accepted_languages): - for lang_tuple in accepted_languages: - lang = lang_tuple[0] - if '-' in lang: - lang = lang.split('-', 1)[0] - return lang - return None - - lang = None - hostname = request.headers['Host'] - if hostname.endswith('.cheat.sh'): - lang = hostname[:-9] - - if 'lang' in request.args: - lang = request.args.get('lang') - - header_accept_language = request.headers.get('Accept-Language', '') - if lang is None and header_accept_language: - lang = _find_supported_language( - _parse_accept_language(header_accept_language)) - - return lang - -def _proxy(*args, **kwargs): - # print "method=", request.method, - # print "url=", request.url.replace('/:shell-x/', ':3000/') - # print "headers=", {key: value for (key, value) in request.headers if key != 'Host'} - # print "data=", request.get_data() - # print "cookies=", request.cookies - # print "allow_redirects=", False - - url_before, url_after = request.url.split('/:shell-x/', 1) - url = url_before + ':3000/' - - if 'q' in request.args: - url_after = '?' + "&".join("arg=%s" % x for x in request.args['q'].split()) - - url += url_after - print(url) - print(request.get_data()) - resp = requests.request( - method=request.method, - url=url, - headers={key: value for (key, value) in request.headers if key != 'Host'}, - data=request.get_data(), - cookies=request.cookies, - allow_redirects=False) - - excluded_headers = ['content-encoding', 'content-length', 'transfer-encoding', 'connection'] - headers = [(name, value) for (name, value) in resp.raw.headers.items() - if name.lower() not in excluded_headers] - - response = Response(resp.content, resp.status_code, headers) - return response - - -@app.route("/", methods=['GET', 'POST']) -@app.route("/", methods=["GET", "POST"]) -def answer(topic=None): - """ - Main rendering function, it processes incoming weather queries. - Depending on user agent it returns output in HTML or ANSI format. - - Incoming data: - request.args - request.headers - request.remote_addr - request.referrer - request.query_string - """ - - user_agent = request.headers.get('User-Agent', '').lower() - html_needed = is_html_needed(user_agent) - options = parse_args(request.args) - - if topic in ['apple-touch-icon-precomposed.png', 'apple-touch-icon.png', 'apple-touch-icon-120x120-precomposed.png'] \ - or (topic is not None and any(topic.endswith('/'+x) for x in ['favicon.ico'])): - return '' - - request_id = request.cookies.get('id') - if topic is not None and topic.lstrip('/') == ':last': - if request_id: - topic = last_query(request_id) - else: - return "ERROR: you have to set id for your requests to use /:last\n" - else: - if request_id: - save_query(request_id, topic) - - if request.method == 'POST': - process_post_request(request, html_needed) - if html_needed: - return redirect("/") - return "OK\n" - - if 'topic' in request.args: - return redirect("/%s" % request.args.get('topic')) - - if topic is None: - topic = ":firstpage" - - if topic.startswith(':shell-x/'): - return _proxy() - #return requests.get('http://127.0.0.1:3000'+topic[8:]).text - - lang = get_answer_language(request) - if lang: - options['lang'] = lang - - ip_address = get_request_ip(request) - if '+' in topic: - not_allowed = LIMITS.check_ip(ip_address) - if not_allowed: - return "429 %s\n" % not_allowed, 429 - - html_is_needed = is_html_needed(user_agent) and not is_result_a_script(topic) - if html_is_needed: - output_format='html' - else: - output_format='ansi' - result, found = cheat_wrapper(topic, request_options=options, output_format=output_format) - if 'Please come back in several hours' in result and html_is_needed: - malformed_response = open(os.path.join(CONFIG["path.internal.malformed"])).read() - return malformed_response +import sys - log_query(ip_address, found, topic, user_agent) - if html_is_needed: - return result - return Response(result, mimetype='text/plain') +from app import app, CONFIG if '--debug' in sys.argv: + # Not all debug mode features are available under `gevent` + # https://github.com/pallets/flask/issues/3825 app.debug = True + if 'CHEATSH_PORT' in os.environ: - PORT = int(os.environ.get('CHEATSH_PORT')) + port = int(os.environ.get('CHEATSH_PORT')) else: - PORT = CONFIG['server.port'] -SRV = WSGIServer((CONFIG['server.bind'], PORT), app) # log=None) -print("Starting server on {}:{}".format(SRV.address[0], SRV.address[1])) -SRV.serve_forever() + port = CONFIG['server.port'] + +srv = WSGIServer((CONFIG['server.bind'], port), app) +print("Starting gevent server on {}:{}".format(srv.address[0], srv.address[1])) +srv.serve_forever() diff --git a/doc/standalone.md b/doc/standalone.md new file mode 100644 index 00000000..4b604012 --- /dev/null +++ b/doc/standalone.md @@ -0,0 +1,100 @@ + +You don't need to install anything, to start using *cheat.sh*. +The only tool that you need is *curl*, which is typically installed +in every system. In the rare cases when *curl* is not installed, +there should be one of its alternatives in the system: *wget*, *wget2*, +*httpie*, *ftp* (with HTTP support), *fetch*, etc. + +There are two cases, when you want to install *cheat.sh* locally: + +1. You plan to use it off-line, without Internet access; +2. You want to use your own cheat sheets (additionally, or as a replacement). + +In this case you need to install cheat.sh locally. + +## How to install cheat.sh locally + +To use cheat.sh offline, you need to: + +1. Install it, +2. Fetch its data sources. + +If you already have the cht.sh cli client locally, +you can use it for the standalone installation. +Otherwise it must be installed first. + +``` + curl https://cht.sh/:cht.sh > ~/bin/cht.sh + chmod +x ~/bin/cht.sh +``` + +Now you can install cheat.sh locally: + +``` + cht.sh --standalone-install +``` + +During the installation process, cheat.sh and its +data sources will be installed locally. + +By default `~/.cheat.sh` is used as the installation +directory. + +![cheat.sh standalone installation](https://user-images.githubusercontent.com/3875145/57986904-ef3f1b80-7a7a-11e9-9531-ef37ec74b03a.png) + +If you don't plan to use Redis for caching, +switch the caching off in the config file: + +``` + $ vim ~/.cheat.sh/etc/config.yaml + cache: + type: none +``` + +or with the environment variable `CHEATSH_CACHE_TYPE=none`. + +## Update cheat sheets + +Cheat sheets are fetched and installed to `~/.cheat.sh/upstream`. +To keep the cheat sheets up to date, +run the `cheat.sh` `update-all` command on regular basis. +Ideally, add it to *cron*: + +``` +0 5 0 0 0 $HOME/.cheat.sh/ve/bin/python $HOME/.cheat.sh/lib/fetch.py update-all +``` + +In this example, all information sources will be updated +each day at 5:00 local time, on regular basis. + +## cheat.sh server mode + +Your local cheat.sh installation is full-fledged, and it can +handle incoming HTTP/HTTPS queries. + +To start cheat.sh in the server mode, run: + +``` +$HOME/.cheat.sh/ve/bin/python $HOME/.cheat.sh/bin/srv.py +``` + +You can also use `gunicorn` to start the cheat.sh server. + + +## Docker + +You can deploy cheat.sh as a docker container. +Use `Dockerfile` in the source root directory, to build the Docker image: + +``` +docker build . +``` + +## Limitations + +Some cheat sheets not available in the offline mode +for the moment. The reason for that is that to process some queries, +cheat.sh needs to access the Internet itself, because it does not have +the necessary data locally. We are working on that how to overcome +this limitation, but for the moment it still exists. + diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index 56c5e9c9..1a4f81fa 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -1,8 +1,19 @@ -# Compose override to add --debug option to bin/srv.py -# call to print tracebacks on errors to stdout. +# Compose override, see https://docs.docker.com/compose/extends/ +# +# - Run `flask` standalone server with more debug aids instead of `gevent` +# - Turn on Flask debug mode to print tracebacks and autoreload code +# - Mounts fresh sources from current dir as volume +# +# Usage: +# docker-compose -f docker-compose.yml -f docker-compose.debug.yml up # -# See https://docs.docker.com/compose/extends/ version: '2' services: app: - command: "--debug" + environment: + FLASK_ENV: development + #FLASK_RUN_RELOAD: False + FLASK_APP: "bin/app.py" + FLASK_RUN_HOST: 0.0.0.0 + FLASK_RUN_PORT: 8002 + entrypoint: ["/usr/bin/flask", "run"] diff --git a/docker-compose.prebuilt.yml b/docker-compose.prebuilt.yml deleted file mode 100644 index 898d9f29..00000000 --- a/docker-compose.prebuilt.yml +++ /dev/null @@ -1,14 +0,0 @@ -version: '2' -services: - app: - image: bglopez/cheat.sh - depends_on: - - redis - ports: - - "8002:8002" - redis: - image: redis:4-alpine - volumes: - - redis_data:/data -volumes: - redis_data: diff --git a/docker-compose.yml b/docker-compose.yml index 9e5f58f2..bba1ddab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,8 @@ services: - CHEATSH_CACHE_REDIS_HOST=redis ports: - "8002:8002" + volumes: + - .:/app:Z redis: image: redis:4-alpine volumes: diff --git a/lib/adapter/adapter.py b/lib/adapter/adapter.py index f33babb0..ffa55ec4 100644 --- a/lib/adapter/adapter.py +++ b/lib/adapter/adapter.py @@ -145,6 +145,7 @@ def get_page_dict(self, topic, request_options=None): 'topic': topic, 'topic_type': self._adapter_name, 'format': self._get_output_format(topic), + 'cache': self._cache_needed, } answer_dict.update(answer) diff --git a/lib/adapter/cmd.py b/lib/adapter/cmd.py index 1e909910..edcf923a 100644 --- a/lib/adapter/cmd.py +++ b/lib/adapter/cmd.py @@ -1,15 +1,14 @@ """ """ -# pylint: disable=relative-import,wrong-import-position,unused-argument,abstract-method +# pylint: disable=unused-argument,abstract-method -from gevent.monkey import patch_all -from gevent.subprocess import Popen, PIPE +import os.path +import re +from subprocess import Popen, PIPE from .adapter import Adapter -import os.path -import re def _get_abspath(path): """Find absolute path of the specified `path` diff --git a/lib/adapter/git_adapter.py b/lib/adapter/git_adapter.py index b8a8d296..0ce4d319 100644 --- a/lib/adapter/git_adapter.py +++ b/lib/adapter/git_adapter.py @@ -134,7 +134,7 @@ def save_state(cls, state): """ local_repository_dir = cls.local_repository_location() state_filename = os.path.join(local_repository_dir, '.cached_revision') - open(state_filename, 'w').write(state) + open(state_filename, 'wb').write(state) @classmethod def get_state(cls): diff --git a/lib/adapter/question.py b/lib/adapter/question.py index 0e22d859..994156d4 100644 --- a/lib/adapter/question.py +++ b/lib/adapter/question.py @@ -4,15 +4,13 @@ path.internal.bin.upstream """ -# pylint: disable=relative-import,wrong-import-position,wrong-import-order +# pylint: disable=relative-import from __future__ import print_function -from gevent.monkey import patch_all -from gevent.subprocess import Popen, PIPE - import os import re +from subprocess import Popen, PIPE from polyglot.detect import Detector from polyglot.detect.base import UnknownLanguage @@ -21,6 +19,25 @@ from languages_data import SO_NAME from .upstream import UpstreamAdapter +NOT_FOUND_MESSAGE = """404 NOT FOUND + +Unknown cheat sheet. Please try to reformulate your query. +Query format: + + /LANG/QUESTION + +Examples: + + /python/read+json + /golang/run+external+program + /js/regex+search + +See /:help for more info. + +If the problem persists, file a GitHub issue at +github.com/chubin/cheat.sh or ping @igor_chubin +""" + class Question(UpstreamAdapter): """ @@ -94,6 +111,10 @@ def _get_page(self, topic, request_options=None): cmd = [CONFIG["path.internal.bin.upstream"]] + topic proc = Popen(cmd, stdin=open(os.devnull, "r"), stdout=PIPE, stderr=PIPE) answer = proc.communicate()[0].decode('utf-8') + + if not answer: + return NOT_FOUND_MESSAGE + return answer def get_list(self, prefix=None): diff --git a/lib/adapter/upstream.py b/lib/adapter/upstream.py index 7622b76e..786d9b22 100644 --- a/lib/adapter/upstream.py +++ b/lib/adapter/upstream.py @@ -48,7 +48,7 @@ class UpstreamAdapter(Adapter): _adapter_name = "upstream" _output_format = "ansi" - _cache_needed = True + _cache_needed = False def _get_page(self, topic, request_options=None): @@ -58,7 +58,7 @@ def _get_page(self, topic, request_options=None): + "?" + options_string try: response = requests.get(url, timeout=CONFIG["upstream.timeout"]) - answer = response.text + answer = {"cache": False, "answer": response.text} except requests.exceptions.ConnectionError: answer = {"cache": False, "answer":_are_you_offline()} return answer diff --git a/lib/cheat_wrapper.py b/lib/cheat_wrapper.py index 73fc74a4..fdcaff67 100644 --- a/lib/cheat_wrapper.py +++ b/lib/cheat_wrapper.py @@ -11,7 +11,7 @@ import re import json -from routing import get_answer_dict, get_topics_list +from routing import get_answers, get_topics_list from search import find_answers_by_keyword from languages_data import LANGUAGE_ALIAS, rewrite_editor_section_name import postprocessing @@ -19,6 +19,18 @@ import frontend.html import frontend.ansi +def _add_section_name(query): + # temporary solution before we don't find a fixed one + if ' ' not in query and '+' not in query: + return query + if '/' in query: + return query + if ' ' in query: + return re.sub(r' +', '/', query, count=1) + if '+' in query: + # replace only single + to avoid catching g++ and friends + return re.sub(r'([^\+])\+([^\+])', r'\1/\2', query, count=1) + def cheat_wrapper(query, request_options=None, output_format='ansi'): """ Function that delivers cheat sheet for `query`. @@ -26,17 +38,6 @@ def cheat_wrapper(query, request_options=None, output_format='ansi'): Additional request options specified in `request_options`. """ - def _add_section_name(query): - # temporary solution before we don't find a fixed one - if ' ' not in query and '+' not in query: - return query - if '/' in query: - return query - if ' ' in query: - # for standalone queries only that may contain ' ' - return "%s/%s" % tuple(query.split(' ', 1)) - return "%s/%s" % tuple(query.split('+', 1)) - def _rewrite_aliases(word): if word == ':bash.completion': return ':bash_completion' @@ -98,7 +99,7 @@ def _parse_query(query): answers = find_answers_by_keyword( topic, keyword, options=search_options, request_options=request_options) else: - answers = [get_answer_dict(topic, request_options=request_options)] + answers = get_answers(topic, request_options=request_options) answers = [ postprocessing.postprocess( diff --git a/lib/cheat_wrapper_test.py b/lib/cheat_wrapper_test.py new file mode 100644 index 00000000..72449aba --- /dev/null +++ b/lib/cheat_wrapper_test.py @@ -0,0 +1,39 @@ +from cheat_wrapper import _add_section_name + +unchanged = """ +python/:list +ls ++ +g++ +g/+ +clang++ +btrfs~volume +:intro +:cht.sh +python/copy+file +python/rosetta/:list +emacs:go-mode/:list +g++g++ +""" + +split = """ +python copy file +python/copy file + +python file +python/file + +python+file +python/file + +g++ -O1 +g++/-O1 +""" + +def test_header_split(): + for inp in unchanged.strip().splitlines(): + assert inp == _add_section_name(inp) + + for test in split.strip().split('\n\n'): + inp, outp = test.split('\n') + assert outp == _add_section_name(inp) diff --git a/lib/fetch.py b/lib/fetch.py index 75a03042..db0c4586 100644 --- a/lib/fetch.py +++ b/lib/fetch.py @@ -57,7 +57,9 @@ def _fetch_locations(known_location): sys.stdout.write("Fetching %s..." % (adptr)) sys.stdout.flush() try: - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + universal_newlines=True) except OSError: print("\nERROR: %s" % cmd) raise @@ -89,6 +91,7 @@ def _fetch_locations(known_location): if os.path.exists(location): if skip_existing: existing_locations.append(location) + print("Already exists %s" % (location)) else: fatal("%s already exists" % location) @@ -136,6 +139,7 @@ def _update_adapter(adptr): updates = [] if cmd: errorcode, output = _run_cmd(cmd) + output = output.decode("utf-8") if errorcode: _log("\nERROR:\n---\n" + output + "\n---\nCould not get list of pages to be updated: %s" % adptr) return False @@ -200,6 +204,10 @@ def main(args): _show_usage() sys.exit(0) + logdir = os.path.dirname(CONFIG["path.log.fetch"]) + if not os.path.exists(logdir): + os.makedirs(logdir) + logging.basicConfig( filename=CONFIG["path.log.fetch"], level=logging.DEBUG, diff --git a/lib/fmt/comments.py b/lib/fmt/comments.py index 824140fd..8bd122d7 100644 --- a/lib/fmt/comments.py +++ b/lib/fmt/comments.py @@ -18,19 +18,15 @@ Configuration parameters: """ -# pylint: disable=wrong-import-position,wrong-import-order - from __future__ import print_function -from gevent.monkey import patch_all -from gevent.subprocess import Popen - import sys import os import textwrap import hashlib import re from itertools import groupby, chain +from subprocess import Popen from tempfile import NamedTemporaryFile from config import CONFIG @@ -234,7 +230,7 @@ def _beautify(text, filetype, add_comments=False, remove_text=False): # or remove the text completely. Otherwise the code has to remain aligned unindent_code = add_comments or remove_text - lines = [x.rstrip('\n') for x in text.splitlines()] + lines = [x.decode("utf-8").rstrip('\n') for x in text.splitlines()] lines = _cleanup_lines(lines) lines_classes = zip(_classify_lines(lines), lines) lines_classes = _wrap_lines(lines_classes, unindent_code=unindent_code) @@ -296,7 +292,8 @@ def beautify(text, lang, options): # if mode is unknown, just don't transform the text at all return text - text = text.encode('utf-8') + if isinstance(text, str): + text = text.encode('utf-8') digest = "t:%s:%s:%s" % (hashlib.md5(text).hexdigest(), lang, mode) # temporary added line that removes invalid cache entries diff --git a/lib/frontend/ansi.py b/lib/frontend/ansi.py index 3880af50..276d58ba 100644 --- a/lib/frontend/ansi.py +++ b/lib/frontend/ansi.py @@ -101,6 +101,10 @@ def _visualize(answers, request_options, search_mode=False): if color_style not in CONFIG['frontend.styles']: color_style = '' + # if there is more than one answer, + # show the source of the answer + multiple_answers = len(answers) > 1 + found = True result = "" for answer_dict in answers: @@ -109,14 +113,16 @@ def _visualize(answers, request_options, search_mode=False): answer = answer_dict['answer'] found = found and not topic_type == 'unknown' - if search_mode and topic != 'LIMITED': + if multiple_answers and topic != 'LIMITED': + section_name = f"{topic_type}:{topic}" + if not highlight: - result += "\n[%s]\n" % topic + result += f"#[{section_name}]\n" else: - result += "\n%s%s %s %s%s\n" % ( - colored.bg('dark_gray'), colored.attr("res_underlined"), - topic, - colored.attr("res_underlined"), colored.attr('reset')) + result += "".join([ + "\n", colored.bg('dark_gray'), colored.attr("res_underlined"), + f" {section_name} ", + colored.attr("res_underlined"), colored.attr('reset'), "\n"]) if answer_dict['format'] in ['ansi', 'text']: result += answer diff --git a/lib/frontend/html.py b/lib/frontend/html.py index c73e96c9..43469d34 100644 --- a/lib/frontend/html.py +++ b/lib/frontend/html.py @@ -5,22 +5,19 @@ path.internal.ansi2html """ -from gevent.monkey import patch_all -from gevent.subprocess import Popen, PIPE - -# pylint: disable=wrong-import-position,wrong-import-order import sys import os import re +from subprocess import Popen, PIPE MYDIR = os.path.abspath(os.path.join(__file__, '..', '..')) sys.path.append("%s/lib/" % MYDIR) +# pylint: disable=wrong-import-position from config import CONFIG from globals import error from buttons import TWITTER_BUTTON, GITHUB_BUTTON, GITHUB_BUTTON_FOOTER import frontend.ansi -# pylint: disable=wrong-import-position,wrong-import-order # temporary having it here, but actually we have the same data # in the adapter module @@ -99,7 +96,7 @@ def _html_wrapper(data): curl_line = "$ curl cheat.sh/" if query == ':firstpage': query = "" - form_html = ('

' + form_html = ('' '%s%s' ' List[str]: """ - Return topic type for `topic` or "unknown" if topic can't be determined. + Return list of topic types for `topic` + or ["unknown"] if topic can't be determined. """ - def __get_topic_type(topic): + def __get_topic_type(topic: str) -> List[str]: + result = [] for regexp, route in self.routing_table: if re.search(regexp, topic): if route in self._adapter: if self._adapter[route].is_found(topic): - return route + result.append(route) else: - return route - return CONFIG["routing.default"] + result.append(route) + if not result: + return [CONFIG["routing.default"]] + + # cut the default route off, if there are more than one route found + if len(result) > 1: + return result[:-1] + return result if topic not in self._cached_topic_type: self._cached_topic_type[topic] = __get_topic_type(topic) @@ -106,62 +115,128 @@ def _get_page_dict(self, query, topic_type, request_options=None): """ Return answer_dict for the `query`. """ - return self._adapter[topic_type]\ .get_page_dict(query, request_options=request_options) - def get_answer_dict(self, topic, request_options=None): + def handle_if_random_request(self, topic): + """ + Check if the `query` is a :random one, + if yes we check its correctness and then randomly select a topic, + based on the provided prefix. + """ - Find cheat sheet for the topic. + + def __select_random_topic(prefix, topic_list): + #Here we remove the special cases + cleaned_topic_list = [ x for x in topic_list if '/' not in x and ':' not in x] + + #Here we still check that cleaned_topic_list in not empty + if not cleaned_topic_list: + return prefix + + random_topic = random.choice(cleaned_topic_list) + return prefix + random_topic + + if topic.endswith('/:random') or topic.lstrip('/') == ':random': + #We strip the :random part and see if the query is valid by running a get_topics_list() + if topic.lstrip('/') == ':random' : + topic = topic.lstrip('/') + prefix = topic[:-7] + + topic_list = [x[len(prefix):] + for x in self.get_topics_list() + if x.startswith(prefix)] + + if '' in topic_list: + topic_list.remove('') + + if topic_list: + # This is a correct formatted random query like /cpp/:random as the topic_list is not empty. + random_topic = __select_random_topic(prefix, topic_list) + return random_topic + else: + # This is a wrongly formatted random query like /xyxyxy/:random as the topic_list is empty + # we just strip the /:random and let the already implemented logic handle it. + wrongly_formatted_random = topic[:-8] + return wrongly_formatted_random + + #Here if not a random requst, we just forward the topic + return topic + + def get_answers(self, topic: str, request_options:Dict[str, str] = None) -> List[Dict[str, Any]]: + """ + Find cheat sheets for the topic. Args: `topic` (str): the name of the topic of the cheat sheet Returns: - answer_dict: the answer dictionary + [answer_dict]: list of answers (dictionaries) """ + + # if topic specified as :, + # cut off + topic_type = "" + if re.match("[^/]+:", topic): + topic_type, topic = topic.split(":", 1) - topic_type = self.get_topic_type(topic) + topic = self.handle_if_random_request(topic) + topic_types = self.get_topic_type(topic) + + # if topic_type is specified explicitly, + # show pages only of that type + if topic_type and topic_type in topic_types: + topic_types = [topic_type] # 'question' queries are pretty expensive, that's why they should be handled # in a special way: # we do not drop the old style cache entries and try to reuse them if possible - if topic_type == 'question': + if topic_types == ['question']: answer = cache.get('q:' + topic) if answer: if isinstance(answer, dict): - return answer - return { + return [answer] + return [{ 'topic': topic, 'topic_type': 'question', 'answer': answer, 'format': 'text+code', - } + }] - answer = self._get_page_dict(topic, topic_type, request_options=request_options) - cache.put('q:' + topic, answer) - return answer + answer = self._get_page_dict(topic, topic_types[0], request_options=request_options) + if answer.get("cache", True): + cache.put('q:' + topic, answer) + return [answer] # Try to find cacheable queries in the cache. # If answer was not found in the cache, resolve it in a normal way and save in the cache - cache_needed = self._adapter[topic_type].is_cache_needed() - if cache_needed: - answer = cache.get(topic) - if not isinstance(answer, dict): - answer = None - if answer: - return answer + answers = [] + for topic_type in topic_types: - answer = self._get_page_dict(topic, topic_type, request_options=request_options) - if isinstance(answer, dict): - if "cache" in answer: - cache_needed = answer["cache"] + cache_entry_name = f"{topic_type}:{topic}" + cache_needed = self._adapter[topic_type].is_cache_needed() - if cache_needed and answer: - cache.put(topic, answer) - return answer + if cache_needed: + answer = cache.get(cache_entry_name) + if not isinstance(answer, dict): + answer = None + if answer: + answers.append(answer) + continue + + answer = self._get_page_dict(topic, topic_type, request_options=request_options) + if isinstance(answer, dict): + if "cache" in answer: + cache_needed = answer["cache"] + + if cache_needed and answer: + cache.put(cache_entry_name, answer) + + answers.append(answer) + + return answers # pylint: disable=invalid-name _ROUTER = Router() get_topics_list = _ROUTER.get_topics_list -get_answer_dict = _ROUTER.get_answer_dict +get_answers = _ROUTER.get_answers diff --git a/lib/search.py b/lib/search.py index dc95aea4..e4beaa38 100644 --- a/lib/search.py +++ b/lib/search.py @@ -22,7 +22,7 @@ import re from config import CONFIG -from routing import get_answer_dict, get_topics_list +from routing import get_answers, get_topics_list def _limited_entry(): return { @@ -100,10 +100,11 @@ def find_answers_by_keyword(directory, keyword, options="", request_options=None if not options_dict["recursive"] and '/' in subtopic: continue - answer_dict = get_answer_dict(topic, request_options=request_options) - answer_text = answer_dict.get('answer', '') - if match(answer_text, keyword, options_dict=options_dict): - answers_found.append(answer_dict) + answer_dicts = get_answers(topic, request_options=request_options) + for answer_dict in answer_dicts: + answer_text = answer_dict.get('answer', '') + if match(answer_text, keyword, options_dict=options_dict): + answers_found.append(answer_dict) if len(answers_found) > CONFIG['search.limit']: answers_found.append( diff --git a/requirements.txt b/requirements.txt index af1981cd..3541734b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ pygments dateutils fuzzywuzzy redis -colored +colored<1.4.3 langdetect cffi polyglot @@ -14,3 +14,5 @@ PyICU pycld2 colorama pyyaml +python-Levenshtein +pytest diff --git a/share/ansi2html.sh b/share/ansi2html.sh index 964ffddd..089e72ae 100644 --- a/share/ansi2html.sh +++ b/share/ansi2html.sh @@ -103,8 +103,8 @@ fi printf '%s' " + -