Browse Source

Merge pull request #12784 from Alchez/feat-py3-dependencies

feat: manage Python 3 compatiblity with dependencies
version-14
gavin 4 years ago
committed by GitHub
parent
commit
a8d0e39fd4
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 229 additions and 196 deletions
  1. +3
    -3
      .github/workflows/ci-tests.yml
  2. +2
    -4
      frappe/commands/site.py
  3. +6
    -5
      frappe/core/doctype/scheduled_job_type/scheduled_job_type.py
  4. +18
    -24
      frappe/database/mariadb/database.py
  5. +20
    -11
      frappe/email/receive.py
  6. +27
    -23
      frappe/integrations/doctype/dropbox_settings/dropbox_settings.py
  7. +18
    -12
      frappe/integrations/doctype/google_calendar/google_calendar.py
  8. +13
    -8
      frappe/integrations/doctype/google_contacts/google_contacts.py
  9. +20
    -13
      frappe/integrations/doctype/google_drive/google_drive.py
  10. +12
    -9
      frappe/utils/xlsxutils.py
  11. +14
    -8
      frappe/website/doctype/website_settings/google_indexing.py
  12. +76
    -76
      requirements.txt

+ 3
- 3
.github/workflows/ci-tests.yml View File

@@ -149,9 +149,9 @@ jobs:
run: |
cp ~/frappe-bench/sites/.coverage ${GITHUB_WORKSPACE}
cd ${GITHUB_WORKSPACE}
pip install coveralls==2.2.0
pip install coverage==4.5.4
coveralls
pip install coveralls==3.0.1
pip install coverage==5.5
coveralls --service=github
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }}

+ 2
- 4
frappe/commands/site.py View File

@@ -676,10 +676,8 @@ def start_ngrok(context):
frappe.init(site=site)

port = frappe.conf.http_port or frappe.conf.webserver_port
public_url = ngrok.connect(port=port, options={
'host_header': site
})
print(f'Public URL: {public_url}')
tunnel = ngrok.connect(addr=str(port), host_header=site)
print(f'Public URL: {tunnel.public_url}')
print('Inspect logs at http://localhost:4040')

ngrok_process = ngrok.get_ngrok_process()


+ 6
- 5
frappe/core/doctype/scheduled_job_type/scheduled_job_type.py View File

@@ -2,14 +2,15 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import json
from datetime import datetime
from typing import Dict, List

import frappe, json
from frappe.model.document import Document
from frappe.utils import now_datetime, get_datetime
from datetime import datetime
from croniter import croniter

import frappe
from frappe.model.document import Document
from frappe.utils import get_datetime, now_datetime
from frappe.utils.background_jobs import enqueue, get_jobs




+ 18
- 24
frappe/database/mariadb/database.py View File

@@ -1,17 +1,13 @@
from __future__ import unicode_literals

import frappe
import warnings

import pymysql
from pymysql.times import TimeDelta
from pymysql.constants import ER, FIELD_TYPE
from pymysql.converters import conversions
from pymysql.constants import ER, FIELD_TYPE
from pymysql.converters import conversions, escape_string

from frappe.utils import get_datetime, cstr, UnicodeWithAttrs
import frappe
from frappe.database.database import Database
from six import PY2, binary_type, text_type, string_types
from frappe.database.mariadb.schema import MariaDBTable
from frappe.utils import UnicodeWithAttrs, cstr, get_datetime


class MariaDBDatabase(Database):
@@ -72,22 +68,20 @@ class MariaDBDatabase(Database):
conversions.update({
FIELD_TYPE.NEWDECIMAL: float,
FIELD_TYPE.DATETIME: get_datetime,
UnicodeWithAttrs: conversions[text_type]
UnicodeWithAttrs: conversions[str]
})

if PY2:
conversions.update({
TimeDelta: conversions[binary_type]
})

if usessl:
conn = pymysql.connect(self.host, self.user or '', self.password or '',
port=self.port, charset='utf8mb4', use_unicode = True, ssl=ssl_params,
conv = conversions, local_infile = frappe.conf.local_infile)
else:
conn = pymysql.connect(self.host, self.user or '', self.password or '',
port=self.port, charset='utf8mb4', use_unicode = True, conv = conversions,
local_infile = frappe.conf.local_infile)
conn = pymysql.connect(
user=self.user or '',
password=self.password or '',
host=self.host,
port=self.port,
charset='utf8mb4',
use_unicode=True,
ssl=ssl_params if usessl else None,
conv=conversions,
local_infile=frappe.conf.local_infile
)

# MYSQL_OPTION_MULTI_STATEMENTS_OFF = 1
# # self._conn.set_server_option(MYSQL_OPTION_MULTI_STATEMENTS_OFF)
@@ -111,7 +105,7 @@ class MariaDBDatabase(Database):
def escape(s, percent=True):
"""Excape quotes and percent in given string."""
# pymysql expects unicode argument to escape_string with Python 3
s = frappe.as_unicode(pymysql.escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`")
s = frappe.as_unicode(escape_string(frappe.as_unicode(s)), "utf-8").replace("`", "\\`")

# NOTE separating % escape, because % escape should only be done when using LIKE operator
# or when you use python format string to generate query that already has a %s
@@ -260,7 +254,7 @@ class MariaDBDatabase(Database):
ADD INDEX `%s`(%s)""" % (table_name, index_name, ", ".join(fields)))

def add_unique(self, doctype, fields, constraint_name=None):
if isinstance(fields, string_types):
if isinstance(fields, str):
fields = [fields]
if not constraint_name:
constraint_name = "unique_" + "_".join(fields)


+ 20
- 11
frappe/email/receive.py View File

@@ -1,18 +1,27 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt

from __future__ import unicode_literals
import datetime
import email
import email.utils
import imaplib
import poplib
import re
import time
from email.header import decode_header

import _socket
import chardet
import six
from six import iteritems, text_type
from six.moves import range
import time, _socket, poplib, imaplib, email, email.utils, datetime, chardet, re
from email_reply_parser import EmailReplyParser
from email.header import decode_header

import frappe
from frappe import _, safe_decode, safe_encode
from frappe.utils import (extract_email_id, convert_utc_to_user_timezone, now,
cint, cstr, strip, markdown, parse_addr)
from frappe.core.doctype.file.file import get_random_filename, MaxFileSizeReachedError
from frappe.core.doctype.file.file import (MaxFileSizeReachedError,
get_random_filename)
from frappe.utils import (cint, convert_utc_to_user_timezone, cstr,
extract_email_id, markdown, now, parse_addr, strip)


class EmailSizeExceededError(frappe.ValidationError): pass
class EmailTimeoutError(frappe.ValidationError): pass
@@ -337,7 +346,7 @@ class EmailServer:
return

self.imap.select("Inbox")
for uid, operation in iteritems(uid_list):
for uid, operation in uid_list.items():
if not uid: continue

op = "+FLAGS" if operation == "Read" else "-FLAGS"
@@ -473,7 +482,7 @@ class Email:
self.html_content += markdown(text_content)

def get_charset(self, part):
"""Detect chartset."""
"""Detect charset."""
charset = part.get_content_charset()
if not charset:
charset = chardet.detect(safe_encode(cstr(part)))['encoding']
@@ -484,7 +493,7 @@ class Email:
charset = self.get_charset(part)

try:
return text_type(part.get_payload(decode=True), str(charset), "ignore")
return str(part.get_payload(decode=True), str(charset), "ignore")
except LookupError:
return part.get_payload()



+ 27
- 23
frappe/integrations/doctype/dropbox_settings/dropbox_settings.py View File

@@ -2,22 +2,23 @@
# Copyright (c) 2015, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import dropbox
import json
import frappe
import os
from urllib.parse import parse_qs, urlparse

import dropbox
from rq.timeouts import JobTimeoutException

import frappe
from frappe import _
from frappe.model.document import Document
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size, get_chunk_site
from frappe.integrations.offsite_backup_utils import (get_chunk_site,
get_latest_backup_file, send_email, validate_file_size)
from frappe.integrations.utils import make_post_request
from frappe.utils import (cint, get_request_site_address,
get_files_path, get_backups_path, get_url, encode)
from frappe.utils.backups import new_backup
from frappe.model.document import Document
from frappe.utils import (cint, encode, get_backups_path, get_files_path,
get_request_site_address, get_url)
from frappe.utils.background_jobs import enqueue
from six.moves.urllib.parse import urlparse, parse_qs
from rq.timeouts import JobTimeoutException
from six import text_type
from frappe.utils.backups import new_backup

ignore_list = [".DS_Store"]

@@ -91,7 +92,10 @@ def backup_to_dropbox(upload_db_backup=True):
dropbox_settings['access_token'] = access_token['oauth2_token']
set_dropbox_access_token(access_token['oauth2_token'])

dropbox_client = dropbox.Dropbox(dropbox_settings['access_token'], timeout=None)
dropbox_client = dropbox.Dropbox(
oauth2_access_token=dropbox_settings['access_token'],
timeout=None
)

if upload_db_backup:
if frappe.flags.create_new_backup:
@@ -127,7 +131,7 @@ def upload_from_folder(path, is_private, dropbox_folder, dropbox_client, did_not
else:
response = frappe._dict({"entries": []})

path = text_type(path)
path = str(path)

for f in frappe.get_all("File", filters={"is_folder": 0, "is_private": is_private,
"uploaded_to_dropbox": 0}, fields=['file_url', 'name', 'file_name']):
@@ -286,11 +290,11 @@ def get_redirect_url():
def get_dropbox_authorize_url():
app_details = get_dropbox_settings(redirect_uri=True)
dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
app_details["app_key"],
app_details["app_secret"],
app_details["redirect_uri"],
{},
"dropbox-auth-csrf-token"
consumer_key=app_details["app_key"],
redirect_uri=app_details["redirect_uri"],
session={},
csrf_token_session_key="dropbox-auth-csrf-token",
consumer_secret=app_details["app_secret"]
)

auth_url = dropbox_oauth_flow.start()
@@ -307,13 +311,13 @@ def dropbox_auth_finish(return_access_token=False):
close = '<p class="text-muted">' + _('Please close this window') + '</p>'

dropbox_oauth_flow = dropbox.DropboxOAuth2Flow(
app_details["app_key"],
app_details["app_secret"],
app_details["redirect_uri"],
{
consumer_key=app_details["app_key"],
redirect_uri=app_details["redirect_uri"],
session={
'dropbox-auth-csrf-token': callback.state
},
"dropbox-auth-csrf-token"
csrf_token_session_key="dropbox-auth-csrf-token",
consumer_secret=app_details["app_secret"]
)

if callback.state or callback.code:


+ 18
- 12
frappe/integrations/doctype/google_calendar/google_calendar.py View File

@@ -2,22 +2,23 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import requests
import googleapiclient.discovery
from datetime import datetime, timedelta
from urllib.parse import quote
import google.oauth2.credentials
import requests
from dateutil import parser
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError

import frappe
from frappe import _
from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
from frappe.model.document import Document
from frappe.utils import get_request_site_address
from googleapiclient.errors import HttpError
from frappe.utils import (add_days, add_to_date, get_datetime,
get_request_site_address, get_time_zone, get_weekdays, now_datetime)
from frappe.utils.password import set_encrypted_password
from frappe.utils import add_days, get_datetime, get_weekdays, now_datetime, add_to_date, get_time_zone
from dateutil import parser
from datetime import datetime, timedelta
from six.moves.urllib.parse import quote
from frappe.integrations.doctype.google_settings.google_settings import get_auth_url

SCOPES = "https://www.googleapis.com/auth/calendar"

@@ -171,7 +172,12 @@ def get_google_calendar_object(g_calendar):
}

credentials = google.oauth2.credentials.Credentials(**credentials_dict)
google_calendar = googleapiclient.discovery.build("calendar", "v3", credentials=credentials)
google_calendar = build(
serviceName="calendar",
version="v3",
credentials=credentials,
static_discovery=False
)

check_google_calendar(account, google_calendar)



+ 13
- 8
frappe/integrations/doctype/google_contacts/google_contacts.py View File

@@ -2,17 +2,17 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import requests
import googleapiclient.discovery

import google.oauth2.credentials
import requests
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError

from frappe.model.document import Document
import frappe
from frappe import _
from googleapiclient.errors import HttpError
from frappe.utils import get_request_site_address
from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
from frappe.model.document import Document
from frappe.utils import get_request_site_address

SCOPES = "https://www.googleapis.com/auth/contacts"

@@ -118,7 +118,12 @@ def get_google_contacts_object(g_contact):
}

credentials = google.oauth2.credentials.Credentials(**credentials_dict)
google_contacts = googleapiclient.discovery.build("people", "v1", credentials=credentials)
google_contacts = build(
serviceName="people",
version="v1",
credentials=credentials,
static_discovery=False
)

return google_contacts, account



+ 20
- 13
frappe/integrations/doctype/google_drive/google_drive.py View File

@@ -2,27 +2,29 @@
# Copyright (c) 2019, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import requests
import googleapiclient.discovery
import google.oauth2.credentials
import os
from urllib.parse import quote

from frappe import _
import google.oauth2.credentials
import requests
from apiclient.http import MediaFileUpload
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError

import frappe
from frappe import _
from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
from frappe.integrations.offsite_backup_utils import (get_latest_backup_file,
send_email, validate_file_size)
from frappe.model.document import Document
from frappe.utils import get_request_site_address
from frappe.utils import (get_backups_path, get_bench_path,
get_request_site_address)
from frappe.utils.background_jobs import enqueue
from six.moves.urllib.parse import quote
from apiclient.http import MediaFileUpload
from frappe.utils import get_backups_path, get_bench_path
from frappe.utils.backups import new_backup
from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
from frappe.integrations.offsite_backup_utils import get_latest_backup_file, send_email, validate_file_size

SCOPES = "https://www.googleapis.com/auth/drive"


class GoogleDrive(Document):

def validate(self):
@@ -126,7 +128,12 @@ def get_google_drive_object():
}

credentials = google.oauth2.credentials.Credentials(**credentials_dict)
google_drive = googleapiclient.discovery.build("drive", "v3", credentials=credentials)
google_drive = build(
serviceName="drive",
version="v3",
credentials=credentials,
static_discovery=False
)

return google_drive, account



+ 12
- 9
frappe/utils/xlsxutils.py View File

@@ -1,18 +1,19 @@
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import frappe
import re
from io import BytesIO

import openpyxl
import xlrd
import re
from openpyxl.styles import Font
from openpyxl import load_workbook
from openpyxl.styles import Font
from openpyxl.utils import get_column_letter
from six import BytesIO, string_types

import frappe

ILLEGAL_CHARACTERS_RE = re.compile(r'[\000-\010]|[\013-\014]|[\016-\037]')


# return xlsx file object
def make_xlsx(data, sheet_name, wb=None, column_widths=None):
column_widths = column_widths or []
@@ -31,12 +32,12 @@ def make_xlsx(data, sheet_name, wb=None, column_widths=None):
for row in data:
clean_row = []
for item in row:
if isinstance(item, string_types) and (sheet_name not in ['Data Import Template', 'Data Export']):
if isinstance(item, str) and (sheet_name not in ['Data Import Template', 'Data Export']):
value = handle_html(item)
else:
value = item

if isinstance(item, string_types) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None):
if isinstance(item, str) and next(ILLEGAL_CHARACTERS_RE.finditer(value), None):
# Remove illegal characters from the string
value = re.sub(ILLEGAL_CHARACTERS_RE, '', value)

@@ -80,12 +81,12 @@ def handle_html(data):

return value


def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=None):
if file_url:
_file = frappe.get_doc("File", {"file_url": file_url})
filename = _file.get_full_path()
elif fcontent:
from io import BytesIO
filename = BytesIO(fcontent)
elif filepath:
filename = filepath
@@ -102,6 +103,7 @@ def read_xlsx_file_from_attached_file(file_url=None, fcontent=None, filepath=Non
rows.append(tmp_list)
return rows


def read_xls_file_from_attached_file(content):
book = xlrd.open_workbook(file_contents=content)
sheets = book.sheets()
@@ -111,6 +113,7 @@ def read_xls_file_from_attached_file(content):
rows.append(sheet.row_values(i))
return rows


def build_xlsx_response(data, filename):
xlsx_file = make_xlsx(data, filename)
# write out response as a xlsx type


+ 14
- 8
frappe/website/doctype/website_settings/google_indexing.py View File

@@ -2,17 +2,18 @@
# Copyright (c) 2020, Frappe Technologies and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
import requests
import googleapiclient.discovery

from urllib.parse import quote

import google.oauth2.credentials
import requests
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError

import frappe
from frappe import _
from googleapiclient.errors import HttpError
from frappe.utils import get_request_site_address
from six.moves.urllib.parse import quote
from frappe.integrations.doctype.google_settings.google_settings import get_auth_url
from frappe.utils import get_request_site_address

SCOPES = "https://www.googleapis.com/auth/indexing"

@@ -82,7 +83,12 @@ def get_google_indexing_object():
}

credentials = google.oauth2.credentials.Credentials(**credentials_dict)
google_indexing = googleapiclient.discovery.build("indexing", "v3", credentials=credentials)
google_indexing = build(
serviceName="indexing",
version="v3",
credentials=credentials,
static_discovery=False
)

return google_indexing



+ 76
- 76
requirements.txt View File

@@ -1,79 +1,79 @@
Babel==2.6.0
beautifulsoup4==4.8.2
bleach-whitelist==0.0.10
bleach==3.3.0
boto3==1.10.18
braintree==3.57.1
chardet==3.0.4
Click==7.0
coverage==4.5.4
croniter==0.3.31
cryptography==3.3.2
dropbox==9.1.0
email-reply-parser==0.5.9
Faker==2.0.4
Babel~=2.9.0
beautifulsoup4~=4.9.3
bleach-whitelist~=0.0.11
bleach~=3.3.0
boto3~=1.17.53
braintree~=4.8.0
chardet~=4.0.0
Click~=7.1.2
coverage~=5.5
croniter~=1.0.11
cryptography~=3.4.7
dropbox~=11.7.0
email-reply-parser~=0.5.12
Faker~=8.1.0
future==0.18.2
gitdb2==2.0.6;python_version<'3.4'
GitPython==2.1.15
git-url-parse==1.2.2
google-api-python-client==1.9.3
google-auth-httplib2==0.0.3
google-auth-oauthlib==0.4.1
google-auth==1.18.0
googlemaps==3.1.1
gunicorn==19.10.0
html2text==2016.9.19
html5lib==1.0.1
ipython==7.14.0
jedi==0.17.2 # not directly required. Pinned to fix upstream issue with ipython.
Jinja2==2.11.3
ldap3==2.7
markdown2==2.4.0
git-url-parse~=1.2.2
gitdb~=4.0.7
GitPython~=3.1.14
google-api-python-client~=2.2.0
google-auth-httplib2~=0.1.0
google-auth-oauthlib~=0.4.4
google-auth~=1.29.0
googlemaps~=4.4.5
gunicorn~=20.1.0
html2text==2020.1.16
html5lib~=1.1
ipython~=7.16.1
jedi==0.17.2 # not directly required. Pinned to fix upstream IPython issue (https://github.com/ipython/ipython/issues/12740)
Jinja2~=2.11.3
ldap3~=2.9
markdown2~=2.4.0
maxminddb-geolite2==2018.703
ndg-httpsclient==0.5.1
num2words==0.5.10
oauthlib==3.1.0
openpyxl==2.6.4
passlib==1.7.3
pdfkit==0.6.1
Pillow>=8.0.0
premailer==3.6.1
psutil==5.7.2
psycopg2-binary==2.8.4
pyasn1==0.4.8
PyJWT==1.7.1
PyMySQL==0.9.3
pyngrok==4.1.6
pyOpenSSL==19.1.0
pyotp==2.3.0
PyPDF2==1.26.0
pypng==0.0.20
PyQRCode==1.2.1
python-dateutil==2.8.1
pytz==2019.3
PyYAML==5.4
rauth==0.7.3
redis==3.5.3
requests-oauthlib==1.3.0
requests==2.23.0
RestrictedPython==5.0
rq>=1.1.0
schedule==0.6.0
semantic-version==2.8.4
simple-chalk==0.1.0
six==1.14.0
sqlparse==0.2.4
stripe==2.40.0
terminaltables==3.1.0
unittest-xml-reporting==2.5.2
urllib3==1.25.9
watchdog==0.8.0
Werkzeug==0.16.1
Whoosh==2.7.4
xlrd==1.2.0
zxcvbn-python==4.4.24
pycryptodome==3.9.8
paytmchecksum==1.7.0
wrapt==1.10.11
razorpay==1.2.0
ndg-httpsclient~=0.5.1
num2words~=0.5.10
oauthlib~=3.1.0
openpyxl~=3.0.7
passlib~=1.7.4
paytmchecksum~=1.7.0
pdfkit~=0.6.1
Pillow~=8.2.0
premailer~=3.8.0
psutil~=5.8.0
psycopg2-binary~=2.8.6
pyasn1~=0.4.8
pycryptodome~=3.10.1
PyJWT~=1.7.1
PyMySQL~=1.0.2
pyngrok~=5.0.5
pyOpenSSL~=20.0.1
pyotp~=2.6.0
PyPDF2~=1.26.0
pypng~=0.0.20
PyQRCode~=1.2.1
python-dateutil~=2.8.1
pytz==2021.1
PyYAML~=5.4.1
rauth~=0.7.3
razorpay~=1.2.0
redis~=3.5.3
requests-oauthlib~=1.3.0
requests~=2.25.1
RestrictedPython~=5.1
rq~=1.8.0
rsa>=4.1 # not directly required, pinned by Snyk to avoid a vulnerability
schedule~=1.1.0
semantic-version~=2.8.5
simple-chalk~=0.1.0
six~=1.15.0
sqlparse~=0.4.1
stripe~=2.56.0
terminaltables~=3.1.0
unittest-xml-reporting~=3.0.4
urllib3~=1.26.4
watchdog~=2.0.2
Werkzeug~=0.16.1
Whoosh~=2.7.4
wrapt~=1.12.1
xlrd~=2.0.1
zxcvbn-python~=4.4.24

Loading…
Cancel
Save