refactor: remove parent, parenttype, parentfield, idx columns from non-child table doctypes

* feat: add parent, parenttype, idx, parentfield columns to doctypes when transitioning from normal -> child table

* fix: remove parent, parenttype, parentfield, idx from DocType DocType

frappe/api.py

@@ -94,7 +94,8 @@ def handle():
 						"data": doc.save().as_dict()
 					})
 
-					if doc.parenttype and doc.parent:
+					# check for child table doctype
+					if doc.get("parenttype"):
 						frappe.get_doc(doc.parenttype, doc.parent).save()
 
 					frappe.db.commit()

frappe/client.py

@@ -128,7 +128,7 @@ def set_value(doctype, name, fieldname, value=None):
 	:param fieldname: fieldname string or JSON / dict with key value pair
 	:param value: value if fieldname is JSON / dict'''
 
-	if fieldname!="idx" and fieldname in frappe.model.default_fields:
+	if fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
 		frappe.throw(_("Cannot edit standard fields"))
 
 	if not value:
@@ -141,14 +141,15 @@ def set_value(doctype, name, fieldname, value=None):
 	else:
 		values = {fieldname: value}
 
-	doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
-	if doc and doc.parent and doc.parenttype:
+	# check for child table doctype
+	if not frappe.get_meta(doctype).istable:
+		doc = frappe.get_doc(doctype, name)
+		doc.update(values)
+	else:
+		doc = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
 		doc = frappe.get_doc(doc.parenttype, doc.parent)
 		child = doc.getone({"doctype": doctype, "name": name})
 		child.update(values)
-	else:
-		doc = frappe.get_doc(doctype, name)
-		doc.update(values)
 
 	doc.save()
 
@@ -162,10 +163,10 @@ def insert(doc=None):
 	if isinstance(doc, str):
 		doc = json.loads(doc)
 
-	if doc.get("parent") and doc.get("parenttype"):
+	if doc.get("parenttype"):
 		# inserting a child record
-		parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
-		parent.append(doc.get("parentfield"), doc)
+		parent = frappe.get_doc(doc.parenttype, doc.parent)
+		parent.append(doc.parentfield, doc)
 		parent.save()
 		return parent.as_dict()
 	else:
@@ -186,10 +187,10 @@ def insert_many(docs=None):
 		frappe.throw(_('Only 200 inserts allowed in one request'))
 
 	for doc in docs:
-		if doc.get("parent") and doc.get("parenttype"):
+		if doc.get("parenttype"):
 			# inserting a child record
-			parent = frappe.get_doc(doc.get("parenttype"), doc.get("parent"))
-			parent.append(doc.get("parentfield"), doc)
+			parent = frappe.get_doc(doc.parenttype, doc.parent)
+			parent.append(doc.parentfield, doc)
 			parent.save()
 			out.append(parent.name)
 		else:

frappe/contacts/doctype/address/address.py

@@ -241,7 +241,7 @@ def address_query(doctype, txt, searchfield, start, page_len, filters):
 			{mcond} {condition}
 		order by
 			if(locate(%(_txt)s, `tabAddress`.name), locate(%(_txt)s, `tabAddress`.name), 99999),
-			`tabAddress`.idx desc, `tabAddress`.name
+			`tabAddress`.name
 		limit %(start)s, %(page_len)s """.format(
 			mcond=get_match_cond(doctype),
 			key=searchfield,

frappe/contacts/doctype/contact/contact.py

@@ -215,7 +215,7 @@ def contact_query(doctype, txt, searchfield, start, page_len, filters):
 			{mcond}
 		order by
 			if(locate(%(_txt)s, `tabContact`.name), locate(%(_txt)s, `tabContact`.name), 99999),
-			`tabContact`.idx desc, `tabContact`.name
+			`tabContact`.name
 		limit %(start)s, %(page_len)s """.format(mcond=get_match_cond(doctype), key=searchfield), {
 			'txt': '%' + txt + '%',
 			'_txt': txt.replace("%", ""),

frappe/core/doctype/data_import/importer.py

@@ -618,7 +618,7 @@ class Row:
 			)
 
 		# remove standard fields and __islocal
-		for key in frappe.model.default_fields + ("__islocal",):
+		for key in frappe.model.default_fields + frappe.model.child_table_fields + ("__islocal",):
 			doc.pop(key, None)
 
 		for col, value in zip(columns, values):

frappe/core/doctype/doctype/doctype.py

@@ -10,7 +10,9 @@ from frappe.cache_manager import clear_user_cache, clear_controller_cache
 import frappe
 from frappe import _
 from frappe.utils import now, cint
-from frappe.model import no_value_fields, default_fields, data_fieldtypes, table_fields, data_field_options
+from frappe.model import (
+	no_value_fields, default_fields, table_fields, data_field_options, child_table_fields
+)
 from frappe.model.document import Document
 from frappe.model.base_document import get_controller
 from frappe.custom.doctype.property_setter.property_setter import make_property_setter
@@ -74,6 +76,7 @@ class DocType(Document):
 		self.make_amendable()
 		self.make_repeatable()
 		self.validate_nestedset()
+		self.validate_child_table()
 		self.validate_website()
 		self.ensure_minimum_max_attachment_limit()
 		validate_links_table_fieldnames(self)
@@ -689,6 +692,23 @@ class DocType(Document):
 		})
 		self.nsm_parent_field = parent_field_name
 
+	def validate_child_table(self):
+		if not self.get("istable") or self.is_new():
+			# if the doctype is not a child table then return
+			# if the doctype is a new doctype and also a child table then
+			# don't move forward as it will be handled via schema
+			return
+
+		self.add_child_table_fields()
+
+	def add_child_table_fields(self):
+		from frappe.database.schema import add_column
+
+		add_column(self.name, "parent", "Data")
+		add_column(self.name, "parenttype", "Data")
+		add_column(self.name, "parentfield", "Data")
+		add_column(self.name, "idx", "Int", length=8, not_null=True, default="0")
+
 	def get_max_idx(self):
 		"""Returns the highest `idx`"""
 		max_idx = frappe.db.sql("""select max(idx) from `tabDocField` where parent = %s""",
@@ -1016,7 +1036,7 @@ def validate_fields(meta):
 				sort_fields = [d.split()[0] for d in meta.sort_field.split(',')]
 
 			for fieldname in sort_fields:
-				if not fieldname in fieldname_list + list(default_fields):
+				if fieldname not in (fieldname_list + list(default_fields) + list(child_table_fields)):
 					frappe.throw(_("Sort field {0} must be a valid fieldname").format(fieldname),
 						InvalidFieldNameError)
 

frappe/core/doctype/file/file.py

@@ -878,8 +878,9 @@ def extract_images_from_html(doc, content, is_private=False):
 		else:
 			filename = get_random_filename(content_type=mtype)
 
-		doctype = doc.parenttype if doc.parent else doc.doctype
-		name = doc.parent or doc.name
+		# attaching a file to a child table doc, attaches it to the parent doc
+		doctype = doc.parenttype if doc.get("parent") else doc.doctype
+		name = doc.get("parent") or doc.name
 
 		_file = frappe.get_doc({
 			"doctype": "File",

frappe/core/doctype/report/report.py

@@ -61,7 +61,7 @@ class Report(Document):
 					delete_permanently=True)
 
 	def get_columns(self):
-		return [d.as_dict(no_default_fields = True) for d in self.columns]
+		return [d.as_dict(no_default_fields=True, no_child_table_fields=True) for d in self.columns]
 
 	@frappe.whitelist()
 	def set_doctype_roles(self):

frappe/core/doctype/user_type/user_type.py

@@ -193,7 +193,7 @@ def get_user_linked_doctypes(doctype, txt, searchfield, start, page_len, filters
 		['DocType', 'read_only', '=', 0], ['DocType', 'name', 'like', '%{0}%'.format(txt)]]
 
 	doctypes = frappe.get_all('DocType', fields = ['`tabDocType`.`name`'], filters=filters,
-		order_by = '`tabDocType`.`idx` desc', limit_start=start, limit_page_length=page_len, as_list=1)
+		limit_start=start, limit_page_length=page_len, as_list=1)
 
 	custom_dt_filters = [['Custom Field', 'dt', 'like', '%{0}%'.format(txt)],
 		['Custom Field', 'options', '=', 'User'], ['Custom Field', 'fieldtype', '=', 'Link']]

frappe/database/database.py

@@ -37,9 +37,9 @@ class Database(object):
 
 	OPTIONAL_COLUMNS = ["_user_tags", "_comments", "_assign", "_liked_by"]
 	DEFAULT_SHORTCUTS = ['_Login', '__user', '_Full Name', 'Today', '__today', "now", "Now"]
-	STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by', 'parent', 'parentfield', 'parenttype')
-	DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus', 'parent',
-		'parentfield', 'parenttype', 'idx']
+	STANDARD_VARCHAR_COLUMNS = ('name', 'owner', 'modified_by')
+	DEFAULT_COLUMNS = ['name', 'creation', 'modified', 'modified_by', 'owner', 'docstatus']
+	CHILD_TABLE_COLUMNS = ('parent', 'parenttype', 'parentfield', 'idx')
 	MAX_WRITES_PER_TRANSACTION = 200_000
 
 	class InvalidColumnName(frappe.ValidationError): pass
@@ -435,11 +435,9 @@ class Database(object):
 
 		else:
 			fields = fieldname
-			if fieldname!="*":
+			if fieldname != "*":
 				if isinstance(fieldname, str):
 					fields = [fieldname]
-				else:
-					fields = fieldname
 
 			if (filters is not None) and (filters!=doctype or doctype=="DocType"):
 				try:

frappe/database/mariadb/framework_mariadb.sql

@@ -171,10 +171,6 @@ CREATE TABLE `tabDocType` (
   `modified_by` varchar(255) DEFAULT NULL,
   `owner` varchar(255) DEFAULT NULL,
   `docstatus` int(1) NOT NULL DEFAULT 0,
-  `parent` varchar(255) DEFAULT NULL,
-  `parentfield` varchar(255) DEFAULT NULL,
-  `parenttype` varchar(255) DEFAULT NULL,
-  `idx` int(8) NOT NULL DEFAULT 0,
   `search_fields` varchar(255) DEFAULT NULL,
   `issingle` int(1) NOT NULL DEFAULT 0,
   `is_tree` int(1) NOT NULL DEFAULT 0,
@@ -228,8 +224,7 @@ CREATE TABLE `tabDocType` (
   `subject_field` varchar(255) DEFAULT NULL,
   `sender_field` varchar(255) DEFAULT NULL,
   `migration_hash` varchar(255) DEFAULT NULL,
-  PRIMARY KEY (`name`),
-  KEY `parent` (`parent`)
+  PRIMARY KEY (`name`)
 ) ENGINE=InnoDB ROW_FORMAT=DYNAMIC CHARACTER SET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
 
 --

frappe/database/mariadb/schema.py

@@ -18,6 +18,17 @@ class MariaDBTable(DBTable):
 		if index_defs:
 			additional_definitions += ',\n'.join(index_defs) + ',\n'
 
+		# child table columns
+		if self.meta.get("istable") or 0:
+			additional_definitions += ',\n'.join(
+				(
+					f"parent varchar({varchar_len})",
+					f"parentfield varchar({varchar_len})",
+					f"parenttype varchar({varchar_len})",
+					"idx int(8) not null default '0'"
+				)
+			) + ',\n'
+
 		# create table
 		query = f"""create table `{self.table_name}` (
 			name varchar({varchar_len}) not null primary key,
@@ -26,12 +37,7 @@ class MariaDBTable(DBTable):
 			modified_by varchar({varchar_len}),
 			owner varchar({varchar_len}),
 			docstatus int(1) not null default '0',
-			parent varchar({varchar_len}),
-			parentfield varchar({varchar_len}),
-			parenttype varchar({varchar_len}),
-			idx int(8) not null default '0',
 			{additional_definitions}
-			index parent(parent),
 			index modified(modified))
 			ENGINE={engine}
 			ROW_FORMAT=DYNAMIC

frappe/database/postgres/framework_postgres.sql

@@ -176,10 +176,6 @@ CREATE TABLE "tabDocType" (
   "modified_by" varchar(255) DEFAULT NULL,
   "owner" varchar(255) DEFAULT NULL,
   "docstatus" smallint NOT NULL DEFAULT 0,
-  "parent" varchar(255) DEFAULT NULL,
-  "parentfield" varchar(255) DEFAULT NULL,
-  "parenttype" varchar(255) DEFAULT NULL,
-  "idx" bigint NOT NULL DEFAULT 0,
   "search_fields" varchar(255) DEFAULT NULL,
   "issingle" smallint NOT NULL DEFAULT 0,
   "is_tree" smallint NOT NULL DEFAULT 0,

frappe/database/postgres/schema.py

@@ -5,26 +5,37 @@ from frappe.database.schema import DBTable, get_definition
 
 class PostgresTable(DBTable):
 	def create(self):
-		add_text = ''
+		add_text = ""
 
 		# columns
 		column_defs = self.get_column_definitions()
-		if column_defs: add_text += ',\n'.join(column_defs)
+		if column_defs:
+			add_text += ",\n".join(column_defs)
+
+		# child table columns
+		if self.meta.get("istable") or 0:
+			if column_defs:
+				add_text += ",\n"
+
+			add_text += ",\n".join(
+				(
+					"parent varchar({varchar_len})",
+					"parentfield varchar({varchar_len})",
+					"parenttype varchar({varchar_len})",
+					"idx bigint not null default '0'"
+				)
+			)
 
 		# TODO: set docstatus length
 		# create table
-		frappe.db.sql("""create table `%s` (
+		frappe.db.sql(("""create table `%s` (
 			name varchar({varchar_len}) not null primary key,
 			creation timestamp(6),
 			modified timestamp(6),
 			modified_by varchar({varchar_len}),
 			owner varchar({varchar_len}),
 			docstatus smallint not null default '0',
-			parent varchar({varchar_len}),
-			parentfield varchar({varchar_len}),
-			parenttype varchar({varchar_len}),
-			idx bigint not null default '0',
-			%s)""".format(varchar_len=frappe.db.VARCHAR_LEN) % (self.table_name, add_text))
+			%s)""" % (self.table_name, add_text)).format(varchar_len=frappe.db.VARCHAR_LEN))
 
 		self.create_indexes()
 		frappe.db.commit()

frappe/database/schema.py

@@ -106,6 +106,9 @@ class DBTable:
 
 		columns = [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
 			frappe.db.STANDARD_VARCHAR_COLUMNS]
+		if self.meta.get("istable"):
+			columns += [frappe._dict({"fieldname": f, "fieldtype": "Data"}) for f in
+				frappe.db.CHILD_TABLE_COLUMNS if f != "idx"]
 		columns += self.columns.values()
 
 		for col in columns:
@@ -300,12 +303,13 @@ def validate_column_length(fieldname):
 def get_definition(fieldtype, precision=None, length=None):
 	d = frappe.db.type_map.get(fieldtype)
 
-	# convert int to long int if the length of the int is greater than 11
+	if not d:
+		return
+
 	if fieldtype == "Int" and length and length > 11:
+		# convert int to long int if the length of the int is greater than 11
 		d = frappe.db.type_map.get("Long Int")
 
-	if not d: return
-
 	coltype = d[0]
 	size = d[1] if d[1] else None
 
@@ -315,19 +319,44 @@ def get_definition(fieldtype, precision=None, length=None):
 		if fieldtype in ["Float", "Currency", "Percent"] and cint(precision) > 6:
 			size = '21,9'
 
-		if coltype == "varchar" and length:
-			size = length
+		if length:
+			if coltype == "varchar":
+				size = length
+			elif coltype == "int" and length < 11:
+				# allow setting custom length for int if length provided is less than 11
+				# NOTE: this will only be applicable for mariadb as frappe implements int
+				# in postgres as bigint (as seen in type_map)
+				size = length
 
 	if size is not None:
 		coltype = "{coltype}({size})".format(coltype=coltype, size=size)
 
 	return coltype
 
-def add_column(doctype, column_name, fieldtype, precision=None):
+def add_column(
+	doctype,
+	column_name,
+	fieldtype,
+	precision=None,
+	length=None,
+	default=None,
+	not_null=False
+):
 	if column_name in frappe.db.get_table_columns(doctype):
 		# already exists
 		return
 
 	frappe.db.commit()
-	frappe.db.sql("alter table `tab%s` add column %s %s" % (doctype,
-		column_name, get_definition(fieldtype, precision)))
+
+	query = "alter table `tab%s` add column %s %s" % (
+		doctype,
+		column_name,
+		get_definition(fieldtype, precision, length)
+	)
+
+	if not_null:
+		query += " not null"
+	if default:
+		query += f" default '{default}'"
+
+	frappe.db.sql(query)

frappe/desk/doctype/tag/tag.py

@@ -148,8 +148,6 @@ def update_tags(doc, tags):
 			"doctype": "Tag Link",
 			"document_type": doc.doctype,
 			"document_name": doc.name,
-			"parenttype": doc.doctype,
-			"parent": doc.name,
 			"title": doc.get_title() or '',
 			"tag": tag
 		}).insert(ignore_permissions=True)

frappe/desk/form/linked_with.py

@@ -389,8 +389,6 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 		else:
 			return results
 
-	me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
-
 	for dt, link in linkinfo.items():
 		filters = []
 		link["doctype"] = dt
@@ -413,11 +411,16 @@ def get_linked_docs(doctype, name, linkinfo=None, for_doctype=None):
 					ret = frappe.get_all(doctype=dt, fields=fields, filters=link.get("filters"))
 
 				elif link.get("get_parent"):
-					if me and me.parent and me.parenttype == dt:
+					ret = None
+
+					# check for child table
+					if not frappe.get_meta(doctype).istable:
+						continue
+
+					me = frappe.db.get_value(doctype, name, ["parenttype", "parent"], as_dict=True)
+					if me and me.parenttype == dt:
 						ret = frappe.get_all(doctype=dt, fields=fields,
 							filters=[[dt, "name", '=', me.parent]])
-					else:
-						ret = None
 
 				elif link.get("child_doctype"):
 					or_filters = [[link.get('child_doctype'), link_fieldnames, '=', name] for link_fieldnames in link.get("fieldname")]
@@ -473,7 +476,7 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
 	ret.update(get_linked_fields(doctype, without_ignore_user_permissions_enabled))
 	ret.update(get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled))
 
-	filters=[['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
+	filters = [['fieldtype', 'in', frappe.model.table_fields], ['options', '=', doctype]]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 	# find links of parents
 	links = frappe.get_all("DocField", fields=["parent as dt"], filters=filters)
@@ -498,12 +501,12 @@ def _get_linked_doctypes(doctype, without_ignore_user_permissions_enabled=False)
 
 def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 
-	filters=[['fieldtype','=', 'Link'], ['options', '=', doctype]]
+	filters = [['fieldtype','=', 'Link'], ['options', '=', doctype]]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 
 	# find links of parents
 	links = frappe.get_all("DocField", fields=["parent", "fieldname"], filters=filters, as_list=1)
-	links+= frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
+	links += frappe.get_all("Custom Field", fields=["dt as parent", "fieldname"], filters=filters, as_list=1)
 
 	ret = {}
 
@@ -529,34 +532,37 @@ def get_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 def get_dynamic_linked_fields(doctype, without_ignore_user_permissions_enabled=False):
 	ret = {}
 
-	filters=[['fieldtype','=', 'Dynamic Link']]
+	filters = [['fieldtype','=', 'Dynamic Link']]
 	if without_ignore_user_permissions_enabled: filters.append(['ignore_user_permissions', '!=', 1])
 
 	# find dynamic links of parents
 	links = frappe.get_all("DocField", fields=["parent as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
-	links+= frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
+	links += frappe.get_all("Custom Field", fields=["dt as doctype", "fieldname", "options as doctype_fieldname"], filters=filters)
 
 	for df in links:
 		if is_single(df.doctype): continue
 
-		# optimized to get both link exists and parenttype
-		possible_link = frappe.get_all(df.doctype, filters={df.doctype_fieldname: doctype},
-		fields=['parenttype'], distinct=True)
+		is_child = frappe.get_meta(df.doctype).istable
+		possible_link = frappe.get_all(
+			df.doctype,
+			filters={df.doctype_fieldname: doctype},
+			fields=["parenttype"] if is_child else None,
+			distinct=True
+		)
 
 		if not possible_link: continue
 
-		for d in possible_link:
-			# is child
-			if d.parenttype:
+		if is_child:
+			for d in possible_link:
 				ret[d.parenttype] = {
 					"child_doctype": df.doctype,
 					"fieldname": [df.fieldname],
 					"doctype_fieldname": df.doctype_fieldname
 				}
-			else:
-				ret[df.doctype] = {
-					"fieldname": [df.fieldname],
-					"doctype_fieldname": df.doctype_fieldname
-				}
+		else:
+			ret[df.doctype] = {
+				"fieldname": [df.fieldname],
+				"doctype_fieldname": df.doctype_fieldname
+			}
 
 	return ret

frappe/desk/reportview.py

@@ -6,7 +6,7 @@
 import frappe, json
 import frappe.permissions
 from frappe.model.db_query import DatabaseQuery
-from frappe.model import default_fields, optional_fields
+from frappe.model import default_fields, optional_fields, child_table_fields
 from frappe import _
 from io import StringIO
 from frappe.core.doctype.access_log.access_log import make_access_log
@@ -156,7 +156,7 @@ def raise_invalid_field(fieldname):
 def is_standard(fieldname):
 	if '.' in fieldname:
 		parenttype, fieldname = get_parenttype_and_fieldname(fieldname, None)
-	return fieldname in default_fields or fieldname in optional_fields
+	return fieldname in default_fields or fieldname in optional_fields or fieldname in child_table_fields
 
 def extract_fieldname(field):
 	for text in (',', '/*', '#'):

frappe/desk/search.py

@@ -147,7 +147,7 @@ def search_widget(doctype, txt, query=None, searchfield=None, start=0,
 			from frappe.model.db_query import get_order_by
 			order_by_based_on_meta = get_order_by(doctype, meta)
 			# 2 is the index of _relevance column
-			order_by = "_relevance, {0}, `tab{1}`.idx desc".format(order_by_based_on_meta, doctype)
+			order_by = "_relevance, {0}".format(order_by_based_on_meta)
 
 			ptype = 'select' if frappe.only_has_select_perm(doctype) else 'read'
 			ignore_permissions = True if doctype == "DocType" else (cint(ignore_user_permissions) and has_permission(doctype, ptype=ptype))

frappe/email/doctype/auto_email_report/auto_email_report.py

@@ -252,7 +252,7 @@ def make_links(columns, data):
 				if col.options and row.get(col.options):
 					row[col.fieldname] = get_link_to_form(row[col.options], row[col.fieldname])
 			elif col.fieldtype == "Currency":
-				doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.parent else None
+				doc = frappe.get_doc(col.parent, doc_name) if doc_name and col.get("parent") else None
 				# Pass the Document to get the currency based on docfield option
 				row[col.fieldname] = frappe.format_value(row[col.fieldname], col, doc=doc)
 	return columns, data

frappe/event_streaming/doctype/document_type_mapping/document_type_mapping.py

@@ -5,7 +5,7 @@ import frappe
 import json
 from frappe import _
 from frappe.model.document import Document
-from frappe.model import default_fields
+from frappe.model import default_fields, child_table_fields
 
 class DocumentTypeMapping(Document):
 	def validate(self):
@@ -14,7 +14,7 @@ class DocumentTypeMapping(Document):
 	def validate_inner_mapping(self):
 		meta = frappe.get_meta(self.local_doctype)
 		for field_map in self.field_mapping:
-			if field_map.local_fieldname not in default_fields:
+			if field_map.local_fieldname not in (default_fields + child_table_fields):
 				field = meta.get_field(field_map.local_fieldname)
 				if not field:
 					frappe.throw(_('Row #{0}: Invalid Local Fieldname').format(field_map.idx))

frappe/model/__init__.py

@@ -90,11 +90,14 @@ default_fields = (
 	'creation',
 	'modified',
 	'modified_by',
+	'docstatus'
+)
+
+child_table_fields = (
 	'parent',
 	'parentfield',
 	'parenttype',
-	'idx',
-	'docstatus'
+	'idx'
 )
 
 optional_fields = (

frappe/model/base_document.py

@@ -4,7 +4,7 @@
 import frappe
 import datetime
 from frappe import _
-from frappe.model import default_fields, table_fields
+from frappe.model import default_fields, table_fields, child_table_fields
 from frappe.model.naming import set_new_name
 from frappe.model.utils.link_count import notify_link_count
 from frappe.modules import load_doctype_module
@@ -104,6 +104,10 @@ class BaseDocument(object):
 				"balance": 42000
 			})
 		"""
+
+		# QUESTION: why do we need the 1st for loop?
+		# we're essentially setting the values in d, in the 2nd for loop (?)
+
 		# first set default field values of base document
 		for key in default_fields:
 			if key in d:
@@ -208,7 +212,10 @@ class BaseDocument(object):
 			raise ValueError
 
 	def remove(self, doc):
-		self.get(doc.parentfield).remove(doc)
+		# Usage: from the parent doc, pass the child table doc
+		# to remove that child doc from the child table, thus removing it from the parent doc
+		if doc.get("parentfield"):
+			self.get(doc.parentfield).remove(doc)
 
 	def _init_child(self, value, key):
 		if not self.doctype:
@@ -285,16 +292,16 @@ class BaseDocument(object):
 			if key not in self.__dict__:
 				self.__dict__[key] = None
 
-			if self.__dict__[key] is None:
-				if key == "docstatus":
-					self.docstatus = DocStatus.draft()
-				elif key == "idx":
-					self.__dict__[key] = 0
+			if key == "docstatus" and self.__dict__[key] is None:
+				self.__dict__[key] = DocStatus.draft()
 
 		for key in self.get_valid_columns():
 			if key not in self.__dict__:
 				self.__dict__[key] = None
 
+			if key == "idx" and self.__dict__[key] is None:
+				self.__dict__[key] = 0
+
 	def get_valid_columns(self):
 		if self.doctype not in frappe.local.valid_columns:
 			if self.doctype in DOCTYPES_FOR_DOCTYPE:
@@ -318,12 +325,19 @@ class BaseDocument(object):
 	def docstatus(self, value):
 		self.__dict__["docstatus"] = DocStatus(cint(value))
 
-	def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False):
+	def as_dict(self, no_nulls=False, no_default_fields=False, convert_dates_to_str=False, no_child_table_fields=False):
 		doc = self.get_valid_dict(convert_dates_to_str=convert_dates_to_str)
 		doc["doctype"] = self.doctype
 		for df in self.meta.get_table_fields():
 			children = self.get(df.fieldname) or []
-			doc[df.fieldname] = [d.as_dict(convert_dates_to_str=convert_dates_to_str, no_nulls=no_nulls, no_default_fields=no_default_fields) for d in children]
+			doc[df.fieldname] = [
+				d.as_dict(
+					convert_dates_to_str=convert_dates_to_str,
+					no_nulls=no_nulls,
+					no_default_fields=no_default_fields,
+					no_child_table_fields=no_child_table_fields
+				) for d in children
+			]
 
 		if no_nulls:
 			for k in list(doc):
@@ -335,6 +349,11 @@ class BaseDocument(object):
 				if k in default_fields:
 					del doc[k]
 
+		if no_child_table_fields:
+			for k in list(doc):
+				if k in child_table_fields:
+					del doc[k]
+
 		for key in ("_user_tags", "__islocal", "__onload", "_liked_by", "__run_link_triggers", "__unsaved"):
 			if self.get(key):
 				doc[key] = self.get(key)
@@ -514,12 +533,12 @@ class BaseDocument(object):
 			if df.fieldtype in table_fields:
 				return "{}: {}: {}".format(_("Error"), _("Data missing in table"), _(df.label))
 
-			elif self.parentfield:
+			# check if parentfield exists (only applicable for child table doctype)
+			elif self.get("parentfield"):
 				return "{}: {} {} #{}: {}: {}".format(_("Error"), frappe.bold(_(self.doctype)),
 					_("Row"), self.idx, _("Value missing for"), _(df.label))
 
-			else:
-				return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
+			return _("Error: Value missing for {0}: {1}").format(_(df.parent), _(df.label))
 
 		missing = []
 
@@ -538,10 +557,11 @@ class BaseDocument(object):
 	def get_invalid_links(self, is_submittable=False):
 		"""Returns list of invalid links and also updates fetch values if not set"""
 		def get_msg(df, docname):
-			if self.parentfield:
+			# check if parentfield exists (only applicable for child table doctype)
+			if self.get("parentfield"):
 				return "{} #{}: {}: {}".format(_("Row"), self.idx, _(df.label), docname)
-			else:
-				return "{}: {}".format(_(df.label), docname)
+
+			return "{}: {}".format(_(df.label), docname)
 
 		invalid_links = []
 		cancelled_links = []
@@ -615,11 +635,8 @@ class BaseDocument(object):
 		fetch_from_fieldname = df.fetch_from.split('.')[-1]
 		value = values[fetch_from_fieldname]
 		if df.fieldtype in ['Small Text', 'Text', 'Data']:
-			if fetch_from_fieldname in default_fields:
-				from frappe.model.meta import get_default_df
-				fetch_from_df = get_default_df(fetch_from_fieldname)
-			else:
-				fetch_from_df = frappe.get_meta(doctype).get_field(fetch_from_fieldname)
+			from frappe.model.meta import get_default_df
+			fetch_from_df = get_default_df(fetch_from_fieldname) or frappe.get_meta(doctype).get_field(fetch_from_fieldname)
 
 			if not fetch_from_df:
 				frappe.throw(
@@ -754,9 +771,9 @@ class BaseDocument(object):
 
 
 	def throw_length_exceeded_error(self, df, max_length, value):
-		if self.parentfield and self.idx:
+		# check if parentfield exists (only applicable for child table doctype)
+		if self.get("parentfield"):
 			reference = _("{0}, Row {1}").format(_(self.doctype), self.idx)
-
 		else:
 			reference = "{0} {1}".format(_(self.doctype), self.name)
 
@@ -867,7 +884,7 @@ class BaseDocument(object):
 		:param parentfield: If fieldname is in child table."""
 		from frappe.model.meta import get_field_precision
 
-		if parentfield and not isinstance(parentfield, str):
+		if parentfield and not isinstance(parentfield, str) and parentfield.get("parentfield"):
 			parentfield = parentfield.parentfield
 
 		cache_key = parentfield or "main"
@@ -894,7 +911,7 @@ class BaseDocument(object):
 		from frappe.utils.formatters import format_value
 
 		df = self.meta.get_field(fieldname)
-		if not df and fieldname in default_fields:
+		if not df:
 			from frappe.model.meta import get_default_df
 			df = get_default_df(fieldname)
 

frappe/model/delete_doc.py

@@ -222,32 +222,36 @@ def check_if_doc_is_linked(doc, method="Delete"):
 	"""
 	from frappe.model.rename_doc import get_link_fields
 	link_fields = get_link_fields(doc.doctype)
-	link_fields = [[lf['parent'], lf['fieldname'], lf['issingle']] for lf in link_fields]
+	ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
+
+	for lf in link_fields:
+		link_dt, link_field, issingle = lf['parent'], lf['fieldname'], lf['issingle']
 
-	for link_dt, link_field, issingle in link_fields:
 		if not issingle:
-			for item in frappe.db.get_values(link_dt, {link_field:doc.name},
-				["name", "parent", "parenttype", "docstatus"], as_dict=True):
-				linked_doctype = item.parenttype if item.parent else link_dt
+			fields = ["name", "docstatus"]
+			if frappe.get_meta(link_dt).istable:
+				fields.extend(["parent", "parenttype"])
 
-				ignore_linked_doctypes = doc.get('ignore_linked_doctypes') or []
+			# NOTE: scenario: parent doc <-(linked to) child table doc <-(linked to) doc
+			for item in frappe.db.get_values(link_dt, {link_field:doc.name}, fields , as_dict=True):
+				# available only in child table cases
+				item_parent = getattr(item, "parent", None)
+				linked_doctype = item.parenttype if item_parent else link_dt
 
 				if linked_doctype in doctypes_to_skip or (linked_doctype in ignore_linked_doctypes and method == 'Cancel'):
 					# don't check for communication and todo!
 					continue
 
-				if not item:
-					continue
-				elif method != "Delete"  and (method != "Cancel" or item.docstatus != 1):
+				if method != "Delete" and (method != "Cancel" or item.docstatus != 1):
 					# don't raise exception if not
 					# linked to a non-cancelled doc when deleting or to a submitted doc when cancelling
 					continue
-				elif link_dt == doc.doctype and (item.parent or item.name) == doc.name:
+				elif link_dt == doc.doctype and (item_parent or item.name) == doc.name:
 					# don't raise exception if not
 					# linked to same item or doc having same name as the item
 					continue
 				else:
-					reference_docname = item.parent or item.name
+					reference_docname = item_parent or item.name
 					raise_link_exists_exception(doc, linked_doctype, reference_docname)
 
 		else:

frappe/model/document.py

@@ -527,7 +527,7 @@ class Document(BaseDocument):
 
 	def _validate_non_negative(self):
 		def get_msg(df):
-			if self.parentfield:
+			if self.get("parentfield"):
 				return "{} {} #{}: {} {}".format(frappe.bold(_(self.doctype)),
 					_("Row"), self.idx, _("Value cannot be negative for"), frappe.bold(_(df.label)))
 			else:
@@ -1202,7 +1202,7 @@ class Document(BaseDocument):
 		if not frappe.compare(val1, condition, val2):
 			label = doc.meta.get_label(fieldname)
 			condition_str = error_condition_map.get(condition, condition)
-			if doc.parentfield:
+			if doc.get("parentfield"):
 				msg = _("Incorrect value in row {0}: {1} must be {2} {3}").format(doc.idx, label, condition_str, val2)
 			else:
 				msg = _("Incorrect value: {0} must be {1} {2}").format(label, condition_str, val2)
@@ -1226,7 +1226,7 @@ class Document(BaseDocument):
 				doc.meta.get("fields", {"fieldtype": ["in", ["Currency", "Float", "Percent"]]}))
 
 		for fieldname in fieldnames:
-			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.parentfield)))
+			doc.set(fieldname, flt(doc.get(fieldname), self.precision(fieldname, doc.get("parentfield"))))
 
 	def get_url(self):
 		"""Returns Desk URL for this document."""
@@ -1379,9 +1379,11 @@ class Document(BaseDocument):
 		doctype = self.__class__.__name__
 
 		docstatus = f" docstatus={self.docstatus}" if self.docstatus else ""
-		parent = f" parent={self.parent}" if self.parent else ""
+		repr_str = f"<{doctype}: {name}{docstatus}"
 
-		return f"<{doctype}: {name}{docstatus}{parent}>"
+		if not hasattr(self, "parent"):
+			return repr_str + ">"
+		return f"{repr_str} parent={self.parent}>"
 
 	def __str__(self):
 		name = self.name or "unsaved"

frappe/model/mapper.py

@@ -4,7 +4,7 @@ import json
 
 import frappe
 from frappe import _
-from frappe.model import default_fields, table_fields
+from frappe.model import default_fields, table_fields, child_table_fields
 from frappe.utils import cstr
 
 
@@ -149,6 +149,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent):
 	no_copy_fields = set([d.fieldname for d in source_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
 		+ [d.fieldname for d in target_doc.meta.get("fields") if (d.no_copy==1 or d.fieldtype in table_fields)]
 		+ list(default_fields)
+		+ list(child_table_fields)
 		+ list(table_map.get("field_no_map", [])))
 
 	for df in target_doc.meta.get("fields"):
@@ -183,7 +184,7 @@ def map_fields(source_doc, target_doc, table_map, source_parent):
 					target_doc.set(fmap[1], val)
 
 	# map idx
-	if source_doc.idx:
+	if source_doc.get("idx"):
 		target_doc.idx = source_doc.idx
 
 	# add fetch

frappe/model/meta.py

@@ -18,7 +18,7 @@ from datetime import datetime
 import click
 import frappe, json, os
 from frappe.utils import cstr, cint, cast
-from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields
+from frappe.model import default_fields, no_value_fields, optional_fields, data_fieldtypes, table_fields, child_table_fields
 from frappe.model.document import Document
 from frappe.model.base_document import BaseDocument
 from frappe.modules import load_doctype_module
@@ -191,6 +191,8 @@ class Meta(Document):
 			else:
 				self._valid_columns = self.default_fields + \
 					[df.fieldname for df in self.get("fields") if df.fieldtype in data_fieldtypes]
+				if self.istable:
+					self._valid_columns += list(child_table_fields)
 
 		return self._valid_columns
 
@@ -520,7 +522,7 @@ class Meta(Document):
 		'''add `links` child table in standard link dashboard format'''
 		dashboard_links = []
 
-		if hasattr(self, 'links') and self.links:
+		if getattr(self, 'links', None):
 			dashboard_links.extend(self.links)
 
 		if not data.transactions:
@@ -625,9 +627,9 @@ def get_field_currency(df, doc=None):
 		frappe.local.field_currency = frappe._dict()
 
 	if not (frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or
-		(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
+		(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))):
 
-		ref_docname = doc.parent or doc.name
+		ref_docname = doc.get("parent") or doc.name
 
 		if ":" in cstr(df.get("options")):
 			split_opts = df.get("options").split(":")
@@ -635,7 +637,7 @@ def get_field_currency(df, doc=None):
 				currency = frappe.get_cached_value(split_opts[0], doc.get(split_opts[1]), split_opts[2])
 		else:
 			currency = doc.get(df.get("options"))
-			if doc.parent:
+			if doc.get("parenttype"):
 				if currency:
 					ref_docname = doc.name
 				else:
@@ -648,7 +650,7 @@ def get_field_currency(df, doc=None):
 				.setdefault(df.fieldname, currency)
 
 	return frappe.local.field_currency.get((doc.doctype, doc.name), {}).get(df.fieldname) or \
-		(doc.parent and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
+		(doc.get("parent") and frappe.local.field_currency.get((doc.doctype, doc.parent), {}).get(df.fieldname))
 
 def get_field_precision(df, doc=None, currency=None):
 	"""get precision based on DocField options and fieldvalue in doc"""
@@ -669,19 +671,25 @@ def get_field_precision(df, doc=None, currency=None):
 
 
 def get_default_df(fieldname):
-	if fieldname in default_fields:
+	if fieldname in (default_fields + child_table_fields):
 		if fieldname in ("creation", "modified"):
 			return frappe._dict(
 				fieldname = fieldname,
 				fieldtype = "Datetime"
 			)
 
-		else:
+		elif fieldname == "idx":
 			return frappe._dict(
 				fieldname = fieldname,
-				fieldtype = "Data"
+				fieldtype = "Int"
 			)
 
+		return frappe._dict(
+			fieldname = fieldname,
+			fieldtype = "Data"
+		)
+
+
 def trim_tables(doctype=None, dry_run=False, quiet=False):
 	"""
 	Removes database fields that don't exist in the doctype (json or custom field). This may be needed
@@ -713,7 +721,7 @@ def trim_tables(doctype=None, dry_run=False, quiet=False):
 
 def trim_table(doctype, dry_run=True):
 	frappe.cache().hdel('table_columns', f"tab{doctype}")
-	ignore_fields = default_fields + optional_fields
+	ignore_fields = default_fields + optional_fields + child_table_fields
 	columns = frappe.db.get_table_columns(doctype)
 	fields = frappe.get_meta(doctype, cached=False).get_fieldnames_with_value()
 	is_internal = lambda f: f not in ignore_fields and not f.startswith("_")

frappe/modules/export_file.py

@@ -47,7 +47,7 @@ def strip_default_fields(doc, doc_export):
 
 	for df in doc.meta.get_table_fields():
 		for d in doc_export.get(df.fieldname):
-			for fieldname in frappe.model.default_fields:
+			for fieldname in (frappe.model.default_fields + frappe.model.child_table_fields):
 				if fieldname in d:
 					del d[fieldname]
 

frappe/public/js/frappe/model/model.js

@@ -10,8 +10,7 @@ $.extend(frappe.model, {
 	layout_fields: ['Section Break', 'Column Break', 'Tab Break', 'Fold'],
 
 	std_fields_list: ['name', 'owner', 'creation', 'modified', 'modified_by',
-		'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus',
-		'parent', 'parenttype', 'parentfield', 'idx'],
+		'_user_tags', '_comments', '_assign', '_liked_by', 'docstatus'],
 
 	core_doctypes_list: ['DocType', 'DocField', 'DocPerm', 'User', 'Role', 'Has Role',
 		'Page', 'Module Def', 'Print Format', 'Report', 'Customize Form',
@@ -20,7 +19,6 @@ $.extend(frappe.model, {
 	std_fields: [
 		{fieldname:'name', fieldtype:'Link', label:__('ID')},
 		{fieldname:'owner', fieldtype:'Link', label:__('Created By'), options: 'User'},
-		{fieldname:'idx', fieldtype:'Int', label:__('Index')},
 		{fieldname:'creation', fieldtype:'Date', label:__('Created On')},
 		{fieldname:'modified', fieldtype:'Date', label:__('Last Updated On')},
 		{fieldname:'modified_by', fieldtype:'Data', label:__('Last Updated By')},

frappe/tests/test_db_update.py

@@ -103,13 +103,9 @@ def get_other_fields_meta(meta):
 	default_fields_map = {
 		'name': ('Data', 0),
 		'owner': ('Data', 0),
-		'parent': ('Data', 0),
-		'parentfield': ('Data', 0),
 		'modified_by': ('Data', 0),
-		'parenttype': ('Data', 0),
 		'creation': ('Datetime', 0),
 		'modified': ('Datetime', 0),
-		'idx': ('Int', 8),
 		'docstatus': ('Check', 0)
 	}
 
@@ -117,8 +113,13 @@ def get_other_fields_meta(meta):
 	if meta.track_seen:
 		optional_fields.append('_seen')
 
+	child_table_fields_map = {}
+	if meta.istable:
+		child_table_fields_map.update({field: ('Data', 0) for field in frappe.db.CHILD_TABLE_COLUMNS if field != 'idx'})
+		child_table_fields_map.update({'idx': ('Int', 8)})
+
 	optional_fields_map = {field: ('Text', 0) for field in optional_fields}
-	fields = dict(default_fields_map, **optional_fields_map)
+	fields = dict(default_fields_map, **optional_fields_map, **child_table_fields_map)
 	field_map =  [frappe._dict({'fieldname': field, 'fieldtype': _type, 'length': _length}) for field, (_type, _length) in fields.items()]
 
 	return field_map

frappe/utils/__init__.py

@@ -24,9 +24,6 @@ import frappe
 from frappe.utils.data import *
 from frappe.utils.html_utils import sanitize_html
 
-default_fields = ['doctype', 'name', 'owner', 'creation', 'modified', 'modified_by',
-	'parent', 'parentfield', 'parenttype', 'idx', 'docstatus']
-
 
 def get_fullname(user=None):
 	"""get the full name (first name + last name) of the user from User"""

frappe/utils/data.py

@@ -1362,7 +1362,7 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) -
 			"fieldtype":
 		}
 	"""
-	from frappe.model import default_fields, optional_fields
+	from frappe.model import default_fields, optional_fields, child_table_fields
 
 	if isinstance(f, dict):
 		key, value = next(iter(f.items()))
@@ -1400,7 +1400,7 @@ def get_filter(doctype: str, f: Union[Dict, List, Tuple], filters_config=None) -
 		frappe.throw(frappe._("Operator must be one of {0}").format(", ".join(valid_operators)))
 
 
-	if f.doctype and (f.fieldname not in default_fields + optional_fields):
+	if f.doctype and (f.fieldname not in default_fields + optional_fields + child_table_fields):
 		# verify fieldname belongs to the doctype
 		meta = frappe.get_meta(f.doctype)
 		if not meta.has_field(f.fieldname):

frappe/website/doctype/web_form/web_form.py

@@ -77,7 +77,7 @@ class WebForm(WebsiteGenerator):
 
 			for prop in docfield_properties:
 				if df.fieldtype==meta_df.fieldtype and prop not in ("idx",
-					"reqd", "default", "description", "default", "options",
+					"reqd", "default", "description", "options",
 					"hidden", "read_only", "label"):
 					df.set(prop, meta_df.get(prop))
 

frappe/website/website_generator.py

@@ -118,7 +118,6 @@ class WebsiteGenerator(Document):
 			"doc": self,
 			"page_or_generator": "Generator",
 			"ref_doctype":self.doctype,
-			"idx": self.idx,
 			"docname": self.name,
 			"controller": get_module_name(self.doctype, self.meta.module),
 		})