Module:CsvUtils - Revision history

76561197969631737: Correcting a few doc mistakes found after copying it into the module/doc page

2023-11-16T00:55:47Z

Correcting a few doc mistakes found after copying it into the module/doc page

← Older revision		Revision as of 00:55, 16 November 2023
Line 24:		Line 24:
	--- lookup of indexes in the data table from string names of the headers. In		--- lookup of indexes in the data table from string names of the headers. In
	--- other words, if you know the column has the header "constructionTime", then		--- other words, if you know the column has the header "constructionTime", then
	--- if the data at index 10 stores the value for constructionTime (45s):		--- if the data at index 10 stores the value for constructionTime, 45s:
	---		---
	--- headerLookupTable["constructionTime"] = 10		--- headerLookupTable["constructionTime"] = 10
Line 32:		Line 32:
	---		---
	--- dataTable = {		--- dataTable = {
	--- [1] = {		--- [1] = {
	--- "header1Name",		--- [1] = "header1Name",
	--- "header2Name",		--- [2] = "header2Name",
	--- "header3Name",		--- [3] = "header3Name",
	--- ...		--- ...
	--- },		--- },
	--- [2] = {		--- [2] = {
	--- [1] = {		--- [1] = {
	--- dataRow1field1,		--- dataRow1field1,
Line 44:		Line 44:
	--- dataRow1field3,		--- dataRow1field3,
	--- ...		--- ...
	--- },		--- },
	--- [2] = {		--- [2] = {
	--- dataRow2field1		--- dataRow2field1,
	--- ...		--- ...
	--- },		--- },
	--- [3] = {		--- [3] = {
	--- dataRow3field1		--- dataRow3field1,
	--- ...		--- ...
	--- },		--- },

76561197969631737: Final code cleanup for today: Reordered methods, labeled regions, and fixed some typos

2023-11-16T00:40:30Z

Final code cleanup for today: Reordered methods, labeled regions, and fixed some typos

Show changes

76561197969631737: Converted comments to proper luadoc comments with three dashes instead of two

2023-11-15T22:29:33Z

Converted comments to proper luadoc comments with three dashes instead of two

Show changes

76561197969631737: Added shortcut method. Moved some commenting to the top to be more visible and significantly improved documentation. Cleaned up some of the code.

2023-11-15T18:30:09Z

Added shortcut method. Moved some commenting to the top to be more visible and significantly improved documentation. Cleaned up some of the code.

Show changes

76561197969631737: Fixing a typo in member function name

2023-11-06T03:45:51Z

Fixing a typo in member function name

← Older revision		Revision as of 03:45, 6 November 2023
Line 91:		Line 91:
	-- getting confused by any special characters like commas or newlines within		-- getting confused by any special characters like commas or newlines within
	-- quotations.		-- quotations.
	encodedString = M.encodeCSV(csvString)		encodedString = encodeCSV(csvString)
	if encodedString == "" then		if encodedString == "" then
	error("Encoding the csv data string removed all information.")		error("Encoding the csv data string removed all information.")

76561197969631737: Commented out the part where single quotes get encoded into HTML characters. this is breaking building names like the Alchemist's Hut

2023-11-06T03:44:45Z

Commented out the part where single quotes get encoded into HTML characters. this is breaking building names like the Alchemist's Hut

← Older revision		Revision as of 03:44, 6 November 2023
Line 4:		Line 4:
	-- automatically removed provided it is enclosed in <noinclude> block. The data		-- automatically removed provided it is enclosed in <noinclude> block. The data
	-- must be wrapped in <pre> to protect line breaks. The <pre> block will also		-- must be wrapped in <pre> to protect line breaks. The <pre> block will also
	-- be removed.		-- be removed. This module also provides a parsing method to convert well-
			-- structured CSV data into a Lua table.
	-- @module CsvUtils		-- @module CsvUtils
	local M = {}		local CsvUtils = {}

	---		---
Line 19:		Line 20:
	-- @return string The raw CSV data.		-- @return string The raw CSV data.
	--		--
	function M.extractCSV(dataPageName)		function CsvUtils.extractCSV(dataPageName)

	-- Load the page and verify something returned correctly. If the page was		-- Load the page and verify something returned correctly. If the page was
Line 79:		Line 80:
	-- @return table A Lua table containing the CSV data.		-- @return table A Lua table containing the CSV data.
	--		--
	function M.luaTableFromCSV(csvString)		function CsvUtils.luaTableFromCSV(csvString)

	-- top level lua table, header row, and data rows (table of rows)		-- top level lua table, header row, and data rows (table of rows)
Line 178:		Line 179:
	-- @return The modified string with special characters replaced.		-- @return The modified string with special characters replaced.
	--		--
	function M.encodeCSV(stringToEncode)		function encodeCSV(stringToEncode)

	-- Call a function on quotations to replace newlines within them. Capture		-- Call a function on quotations to replace newlines within them. Capture
Line 193:		Line 194:

	-- Sub single quotes for their HTML code.		-- Sub single quotes for their HTML code.
	encodedString = encodedString:gsub("'", "'")		--encodedString = encodedString:gsub("'", "'")
	if encodedString == "" then		--if encodedString == "" then
	error("Encoding single quotation marks resulted in no data left.")		-- error("Encoding single quotation marks resulted in no data left.")
	end		--end

	return encodedString		return encodedString
Line 227:		Line 228:
	end		end

	return M		return CsvUtils

76561197969631737: added return M statement to end so it can be required by other modules

2023-11-01T12:18:29Z

added return M statement to end so it can be required by other modules

← Older revision		Revision as of 12:18, 1 November 2023
Line 226:		Line 226:
	return encodedString		return encodedString
	end		end

			return M

76561197969631737: missed a "M" that should be an "M."

2023-11-01T04:19:03Z

missed a "M" that should be an "M."

← Older revision		Revision as of 04:19, 1 November 2023
Line 90:		Line 90:
	-- getting confused by any special characters like commas or newlines within		-- getting confused by any special characters like commas or newlines within
	-- quotations.		-- quotations.
	encodedString = p.encodeCSV(csvString)		encodedString = M.encodeCSV(csvString)
	if encodedString == "" then		if encodedString == "" then
	error("Encoding the csv data string removed all information.")		error("Encoding the csv data string removed all information.")

76561197969631737: Created utility module to retrieve csv data template pages and parse them into lua tables.

2023-11-01T04:11:24Z

Created utility module to retrieve csv data template pages and parse them into lua tables.

New page

---
-- Module for processing retrieving and returning data stored in CSV format in
-- wiki templates. There can be documentation in the template that will be
-- automatically removed provided it is enclosed in <noinclude> block. The data
-- must be wrapped in <pre> to protect line breaks. The <pre> block will also
-- be removed.
-- @module CsvUtils
local M = {}

---
-- Loads the data from a wiki page (usually a template). This method removes
-- everything from the page content except the raw data, including anything
-- enclosed in <noinclude> and including the enclosing <pre> tags that protect
-- line breaks in the raw data. Returns nil if the page could not be loaded or
-- if there was a problem processing the page content.
--
-- @param dataPageName (string) The name of the page containing the CSV data,
-- including the namespace, like this: 'Template:Workshops_Recipes_csv'
-- @return string The raw CSV data.
--
function M.extractCSV(dataPageName)

-- Load the page and verify something returned correctly. If the page was
-- not found, then the variable will be nil or the page won't exist.
local csvPage = mw.title.new(dataPageName)
if not csvPage or not csvPage.exists then
error("Could not find data page: " .. dataPageName)
end

-- Get the content of the page, or it will be nil if there is no page. We
-- can also expect there's a problem if there's an empty string.
local pageContent = csvPage:getContent()
if not pageContent or pageContent == "" then
error("Data page content does not exist: " .. dataPageName)
end

-- Process the content of the full wikipage, expanding templates and
-- applying wikimarkup. If it didn't work or returns an empty string, then
-- there's a problem.
local frame = mw.getCurrentFrame()
local preprocessedText = frame:preprocess(pageContent)
if not preprocessedText or preprocessedText == "" then
error("Data page content could not be preprocessed " .. dataPageName)
end

-- Remove everything within and including the <noinclude> tags. If this
-- results in an empty string, there was a problem. Since we're still
-- dealing with mediawiki content, use mw.ustring instead of Lua string
-- library.
local trimmedText = mw.ustring.gsub(pageContent, "<noinclude>.-</noinclude>", "")
if trimmedText == "" then
error("Trimming the data page content resulted in no csv data left: " .. dataPageName)
end

-- Extract the data from the enclosing <pre> block, leaving only the csv
-- data. If this results in an empty string, there was a problem.
local rawCSV = trimmedText:match("<pre>(.-)</pre>")
if rawCSV == "" then
error("Trimming the data page content resulted in no csv data left: " .. dataPageName)
end

return rawCSV
end -- extractCSV function

---
-- Converts a CSV data string into a Lua table. The CSV data should have a
-- header row with field names. Each row represents a new record, and each
-- column contains specific data for that record. The resulting Lua table has
-- sub-tables for each row. Indices can be used or the names of the header row
-- will be usable as keys to access the fields.
-- For example:
-- row[1] is the same as row["id"] (if the first header in the header row
-- contains the string "id")
--
-- @function luaTableFromCSV
-- @param csvString (string) The CSV data as a string.
-- @return table A Lua table containing the CSV data.
--
function M.luaTableFromCSV(csvString)

-- top level lua table, header row, and data rows (table of rows)
local luaTable = {}
local header = {}
local rows = {}
local headerLookup = {} -- stores header indices for constant time lookup

-- Start by cleaning (encoding) the string up so it can be processed without
-- getting confused by any special characters like commas or newlines within
-- quotations.
encodedString = p.encodeCSV(csvString)
if encodedString == "" then
error("Encoding the csv data string removed all information.")
end

-- Separate the csv data into lines/rows by gmatch'ing on carriage returns
-- and/or newlines. Then make each line a new row (or the header row).
for line in encodedString:gmatch("[^\r\n]+") do
if not line or line == "" then
error("Splitting csv data into rows failed.")
end

-- We only build the header once; then its length will be > 0
-- The first line will be the header, and after we build that,
-- everything else goes into the rows table.
if #header == 0 then
-- Count the headers to use to populate a header lookup table
local i = 1
-- Separate the header row into fields by gmatch'ing again, this
-- time on commas. We can use plus here because there are no empty
-- header cells.
for field in line:gmatch("([^,]+)") do
if not field or field == "" then
error("Splitting row into fields failed on header row.")
end
-- Then append each split field to the header row.
header[i] = field
headerLookup[field] = i
i = i + 1
end

-- Header row done; add it after all the other lines are processed.
if #header == 0 then
error("Adding fields was not successful to header row.")
end

else
-- Build each row separately as a new table.
local row = {}
-- Keep track of indices so fields can be named with header row
-- names.
local i = 1

-- Separate the line into fields by gmatch'ing on commas again. Use
-- an asterisk here instead of a plus because there are empty cells
-- with zero length. The rows must end in a comma to do this
-- correctly or there will be empty strings added between every
-- field in the resulting table.
for field in line:gmatch("([^,]*),") do
-- Empty strings are allowed here.
if not field then
error("Splitting row into fields failed on field number: " .. i
.. " on non-header row number: " .. #rows+1)
end
-- Append to the new row by assigning the key.
row[i] = field
i = i + 1
end

-- Row done. Append the new row to the rows table now.
if #row == 0 then
error("Adding fields was not successful to row number: " .. i)
end
table.insert(rows, row)

end
end

-- Now put the header and rows into the luaTable
if #rows == 0 then
error("There are no rows to add to the table.")
end

table.insert(luaTable, header)
table.insert(luaTable, rows)

return luaTable, headerLookup
end -- luaTableFromCSV function

---
-- Replaces apostrophes and double quotes in a string with their respective
-- HTML character codes.
--
-- @param stringToEncode The input string to be processed.
-- @return The modified string with special characters replaced.
--
function M.encodeCSV(stringToEncode)

-- Call a function on quotations to replace newlines within them. Capture
-- the entire quote, including the bounding quotation marks.
local encodedString = stringToEncode:gsub("(\".-\")", encodeQuotations)

-- Now that the content within quotes are handled:

-- Sub double quotes for their HTML code.
encodedString = encodedString:gsub("\"", """)
if encodedString == "" then
error("Encoding double quotation marks resulted in no data left.")
end

-- Sub single quotes for their HTML code.
encodedString = encodedString:gsub("'", "'")
if encodedString == "" then
error("Encoding single quotation marks resulted in no data left.")
end

return encodedString
end

---
-- Local function to replace newlines that occur within quoted strings with
-- their HTML codes, so that they aren't interpreted as newlines that separate
-- records in the csv data string.
--
-- @param stringToEncode The input string to be processed.
-- @return The modified string with newlines replaced.
function encodeQuotations(stringToEncode)

-- Sub out newlines within this matched quotation
local encodedString = stringToEncode:gsub('[\r\n]+', '
')
if encodedString == "" then
error("Encoding newlines inside quoted descriptions resulted in no data left.")
end

-- Sub out commas within the matched quotation
local encodedString = encodedString:gsub(",", ",")
if encodedString == "" then
error("Encoding commas inside quoted descriptions resulted in no data left.")
end

return encodedString
end