From 6ac7d3fc2e0e2374838ee893afbebe570560c1e8 Mon Sep 17 00:00:00 2001 From: David Roe Date: Sun, 19 Jul 2026 12:02:46 -0400 Subject: [PATCH] Document grouped columns in search downloads (LMFDB#6477) Search-result downloads describe each column in a header comment and a definitions section at the bottom of the file. For sections whose columns are grouped with a ColGroup and downloaded as a nested list of subcolumn values (the abstract subgroup search: Subgroup/Ambient/Quotient), the header listed only the group titles (e.g. [Label, Subgroup, Ambient, Quotient]) and the definitions loop skipped the groups entirely (their knowl is None), so the individual subcolumns were never described. Add SearchCol.download_subcols(info) (overridden in ColGroup) that returns a group's subcolumns exactly when it downloads as a nested list (download_col is None and subcols is a concrete list); ColGroups that set download_col (cmf traces/atkin_lehner, cc power_cols) download as a single value and are unchanged. In downloader.py, use it to (a) spell out each grouped column's subcolumn titles under the "Each entry ... has the form" comment and (b) expand each grouped column in the definitions section, documenting every subcolumn by its short title, name and knowl. Verified by generating sage/magma/gp/text downloads for the subgroup search (now documents all 18/2/11 subcolumns matching the data), plus regressions for cmf newforms, conjugacy classes and number fields (byte-identical headers); the downloaded sage/gp/magma files parse and run. cmf + abstract-group + number field download tests pass; pyflakes/ruff clean. Co-Authored-By: Claude Fable 5 --- lmfdb/utils/downloader.py | 62 +++++++++++++++++++++++++++-------- lmfdb/utils/search_columns.py | 24 ++++++++++++++ 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/lmfdb/utils/downloader.py b/lmfdb/utils/downloader.py index 3ec88c87f7..c6fc323aaf 100644 --- a/lmfdb/utils/downloader.py +++ b/lmfdb/utils/downloader.py @@ -770,7 +770,23 @@ def __call__(self, info): seen.add(name) cols = [cols[i] for i in include] column_names = [column_names[i] for i in include] - data_format = [(col.title if isinstance(col.title, str) else col.title(info)) for col in cols] + + def col_disp_title(col, prefer_short=False): + # The (short) title used to describe a column in the download header and definitions. + title = col.short_title if prefer_short else col.title + if title is None: + title = col.title + return title if isinstance(title, str) else title(info) + + data_format = [col_disp_title(col) for col in cols] + # ColGroups without a download_col are downloaded as a nested list of their subcolumns + # (see ColGroup.download), so we record those subcolumns in order to describe them (#6477). + group_expansions = [] # list of (group_title, [subcolumn_titles]) + for col in cols: + subcols = col.download_subcols(info) + if subcols: + group_expansions.append( + (col_disp_title(col), [col_disp_title(sub, prefer_short=True) for sub in subcols])) first50 = [[col.download(rec) for col in cols] for rec in first50] if num_results > 10000: # Estimate the size of the download file. This won't necessarily be a great estimate @@ -799,6 +815,12 @@ def make_download(): # We then describe the columns included, both in a comment and as a variable yield lang.comment(' Each entry in the following data list has the form:\n') yield lang.comment(' [' + ', '.join(data_format) + ']\n') + # Grouped columns are downloaded as a nested list of their subcolumns, so we spell + # out the contents of each such list here (#6477). + if group_expansions: + yield lang.comment(' where the grouped columns are themselves lists:\n') + for gtitle, subtitles in group_expansions: + yield lang.comment(' %s = [%s]\n' % (gtitle, ', '.join(subtitles))) yield lang.comment(' For more details, see the definitions at the bottom of the file.\n') if make_data_comment: yield lang.comment(f'\n {make_data_comment}\n') @@ -824,8 +846,13 @@ def make_download(): yield "\n" + lang.func_start("make_data", "") + self.makedata_code(lang) + lang.function_end + "\n\n" # We need to be able to look up knowls within knowls, so to reduce the number of database calls we just get them all. - # We do some global preprocessing to get access to knowls that define the columns - if any(col.download_desc is None for col in cols): + # We do some global preprocessing to get access to knowls that define the columns. + # Grouped columns (ColGroups downloaded as a nested list) are expanded into their + # subcolumns so that each subcolumn is documented individually (#6477). + doc_plan = [(col, name, col.download_subcols(info)) for col, name in zip(cols, column_names)] + if any(c.download_desc is None + for col, name, subcols in doc_plan + for c in [col, *subcols]): from lmfdb.knowledge.knowl import knowldb all_knowls = {rec["id"]: (rec["title"], rec["content"]) for rec in knowldb.get_all_knowls(fields=["id", "title", "content"])} knowl_re = re.compile(r"""\{\{\s*KNOWL\(\s*["'](?:[^"']+)["'],\s*(?:title\s*=\s*)?['"]([^"']+)['"]\s*\)\s*\}\}""") @@ -839,12 +866,13 @@ def defines_subber(match): word = match.group(1) return f"**{word}**" - # If we haven't specified a more specific download_desc, we use the column knowl to get a string to add to the bottom of the file for each column - for col, name in zip(cols, column_names): + # If we haven't specified a more specific download_desc, we use the column knowl to get a + # string to add to the bottom of the file for each column + def emit_col_doc(col, name, disp_title): if col.download_desc is None: knowldata = all_knowls.get(col.knowl) if knowldata is None: - continue + return # We want to remove KNOWL and DEFINES macros _, content = knowldata knowl = knowl_re.sub(knowl_subber, content) @@ -854,14 +882,10 @@ def defines_subber(match): else: knowl = col.download_desc if knowl: - if isinstance(col.title, str): - title = col.title - else: - title = col.title(info) - if name.lower() == title.lower(): - yield lang.comment(f" {title} --\n") + if name.lower() == disp_title.lower(): + yield lang.comment(f" {disp_title} --\n") else: - yield lang.comment(f"{title} ({name}) --\n") + yield lang.comment(f"{disp_title} ({name}) --\n") for line in knowl.split("\n"): if line.strip(): yield lang.comment(" " + line.rstrip() + "\n") @@ -869,4 +893,16 @@ def defines_subber(match): yield lang.comment("\n") yield lang.comment("\n\n") + for col, name, subcols in doc_plan: + if subcols: + # A grouped column, downloaded as a list; introduce it and document each subcolumn. + gtitle = col_disp_title(col) + yield lang.comment(f" {gtitle} is a grouped column, downloaded as a list of the following subcolumns:\n\n") + yield from emit_col_doc(col, name, gtitle) + for sub in subcols: + subname = sub.name if sub.download_col is None else sub.download_col + yield from emit_col_doc(sub, subname, col_disp_title(sub, prefer_short=True)) + else: + yield from emit_col_doc(col, name, col_disp_title(col)) + return self._wrap_generator(make_download(), filename, lang=lang) diff --git a/lmfdb/utils/search_columns.py b/lmfdb/utils/search_columns.py index eec9333136..f5d62bc2fb 100644 --- a/lmfdb/utils/search_columns.py +++ b/lmfdb/utils/search_columns.py @@ -228,6 +228,20 @@ def download(self, rec): name = self.download_col return self._get(rec, name=name, downloading=True) + def download_subcols(self, info): + """ + Most columns produce a single value when downloading, so this returns an empty list. + + ``ColGroup`` overrides this: when it downloads as a nested list of its subcolumns' + values (see ``ColGroup.download``), it returns that list of subcolumns so that the + downloader can document each one individually. + + INPUT: + + - ``info`` -- the dictionary created from the url + """ + return [] + class SpacerCol(SearchCol): """ @@ -501,6 +515,16 @@ def download(self, rec): return self._get(rec, name=self.download_col, downloading=True) return [sub.download(rec) for sub in self.subcols] + def download_subcols(self, info): + # When a download_col is set, this group downloads as a single value (see download), + # so we behave like an ordinary column. Otherwise download produces a nested list + # over self.subcols, so we return those subcolumns to be documented individually. + # We only expand a concrete list of subcolumns (matching what download iterates over); + # if subcols is a callable we fall back to documenting the group as a whole. + if self.download_col is not None or callable(self.subcols): + return [] + return self.subcols + class SearchColumns: """