Module:GetData

Documentation for this module may be created at Module:GetData/doc

local p = {}

function p.remoteGet(frame)

    local prop = frame:getParent().args[1]
    local source = frame.args.source
    local title = mw.title.getCurrentTitle().subpageText

    local queryPart = "[[" .. title .. "]]|?" .. prop .. "#|limit=1|valuesep=;|sep=;"
    local encodedQuery = mw.uri.encode(queryPart)
    local queryUrl = "https://" .. source .. ".tsadra.org/api.php?action=ask&api_version=3&format=json&query=" .. encodedQuery
    -- if queryUrl then
    --     return queryUrl
    -- end
    local data, errors = mw.ext.externalData.getExternalData(queryUrl)

    local json = not errors and data.__json or false

    if #json then
        for _, result in ipairs(json.query.results) do
            for name, values in pairs (result) do
                for prop, value in pairs (values.printouts) do
                    local dataOutput = value
                    if type( value ) == 'table' then
                        local flat = {}
                        for _, item in ipairs( value ) do
                            if type( item ) == 'table' then
                                table.insert( flat, item.fulltext or item.label or tostring( item ) )
                            else
                                table.insert( flat, tostring( item ) )
                            end
                        end
                        dataOutput = table.concat( flat, '; ')
                    end
                    return dataOutput
                end
            end
        end
    else
        return 'No data! ¯\\_(ツ)_/¯'
    end

end


function p.remoteGetNonApi(frame)

    local prop = frame:getParent().args[1]
    local source = frame.args.source
    local title = mw.uri.encode( mw.title.getCurrentTitle().subpageText )

    local queryUrl = 'https://' .. source .. '.tsadra.org/index.php?title=Special%3AAsk&q=%5B%5B' .. title .. '%5D%5D&po=' .. prop .. '&p%5Blimit%5D=1&p%5Bformat%5D=csv&p%5Bvaluesep%5D=;'
    -- if queryUrl then
    --     return queryUrl
    -- end
    local data = mw.ext.externalData.getExternalData{
        url = queryUrl ,
        data = {
            Value = prop
        } ,
        format = 'CSV with header'
    }

    if data then
        return queryUrl .. '<br><br>' .. data.Value
    else
        return 'No data! ¯\\_(ツ)_/¯'
    end

end



function p.remoteSet(frame)

    local pargs = frame:getParent().args
	if pargs.page then
		pagename = pargs.page
	else
		pagename = mw.title.getCurrentTitle().subpageText
	end

    local properties = ''
    for value in string.gmatch(pargs.props, "([^;]+)") do
        properties = properties .. "|?" .. value .. "#"
    end

    local extQuery = mw.uri.encode( "[[" .. pagename .. "]]" .. properties .. "|limit=10000|valuesep=;|sep=;" )

    -- local testQuery = "https://" .. pargs.source .. ".tsadra.org/api.php?action=ask&query=" .. extQuery .. "&format=json&api_version=3"
    -- if testQuery then
    --     return testQuery
    -- end

    local remoteContent, rcErrors = mw.ext.externalData.getExternalData("https://" .. pargs.source .. ".tsadra.org/api.php?action=ask&query=" .. extQuery .. "&format=json&api_version=3")
    local json = not rcErrors and remoteContent.__json or false

    if rcErrors then
        return rcErrors
    end
    -- EXAMPLE: https://research.tsadra.org/api.php?action=ask&query=%5B%5BRhoton%2C+J.%5D%5D%7C%3FFirstImage%7C%3FMainNamePhon%7Climit%3D10000&format=json&api_version=3

    -- Properties listed in |multi= are multi-valued: they get a '+sep=' directive so
    -- SMW splits the stored string into separate values. Without |multi=, |sep= applies
    -- to every property in the batch (legacy behaviour, kept for existing callers).
    -- |multi= is what lets one call handle both single- and multi-valued properties,
    -- instead of splitting into two calls (= two remote HTTP fetches).
    local multiProps = {}
    if pargs.multi then
        for value in string.gmatch(pargs.multi, "([^;]+)") do
            multiProps[ mw.text.trim( value ) ] = true
        end
    end

    local separator = pargs.sep or ';'

    local dataStore = {}

    if #json then
        for _, result in ipairs(json.query.results) do
            for name, values in pairs (result) do
                for prop, value in pairs (values.printouts) do
                    local dataOutput = value
                    if type( value ) == 'table' then
                        local flat = {}
                        for _, item in ipairs( value ) do
                            if type( item ) == 'table' then
                                table.insert( flat, item.fulltext or item.label or tostring( item ) )
                            else
                                table.insert( flat, tostring( item ) )
                            end
                        end
                        dataOutput = table.concat( flat, '; ')
                    end
                    table.insert(dataStore, prop .. '=' .. dataOutput)
                    if pargs.multi then
                        -- Only flag the properties explicitly declared multi-valued.
                        if multiProps[prop] then
                            table.insert(dataStore, '+sep=' .. separator)
                        end
                    elseif pargs.sep then
                        -- Legacy: no |multi= given, so |sep= applies to the whole batch.
                        table.insert(dataStore, '+sep=' .. pargs.sep)
                    end
                end
            end
        end
    end

    local dataStored = mw.smw.set( dataStore )

    if dataStored == true then
        return ''
    else
        return 'An error occurred during the storage process. Message reads ' .. dataStored.error
    end

end


function p.set(frame)

    if not mw.smw then
        return "mw.smw module not found"
    end

    local pSetData = mw.loadData('Module:GetData/'..frame.args['data'])
    local parentArgs = frame:getParent().args

    local props = ''
    if parentArgs.props then
        props = parentArgs.props
    else
        props = parentArgs.prop
    end
    
    local dataStore = {}

    for propValue in string.gmatch(props, "([^;]+)") do
        if propValue and pSetData[propValue] then 
            table.insert(dataStore, propValue .. '=' .. pSetData[propValue])
            if parentArgs.sep then
                table.insert(dataStore, '+sep=' .. parentArgs.sep)
            end
        end
    end

    local dataStored = mw.smw.set( dataStore )

    if dataStored == true then
        return ''
    else
        return 'An error occurred during the storage process. Message reads ' .. dataStored.error
    end

end

function p.remoteGetCategory(frame)

    local category = frame.args.category or "Terms"
    local source = frame.args.source or "commons"
    local limit = frame.args.limit or "1000"

    local nocache = frame.args.nocache

    local queryPart = "[[Category:" .. category .. "]]|limit=" .. limit
    local encodedQuery = mw.uri.encode(queryPart)
    local queryUrl = "https://" .. source .. ".tsadra.org/api.php?action=ask&api_version=3&format=json&query=" .. encodedQuery
    if nocache then
        queryUrl = queryUrl .. "&nocache=" .. os.time()
    end

    local data, errors = mw.ext.externalData.getExternalData(queryUrl)

    if errors or not data then
        return 'fetch error'
    end

    local titles = {}
    for i = 1, data.count do
        local row = data[i]
        if type(row) == 'table' and row.fulltext then
            table.insert(titles, row.fulltext)
        end
    end
    return table.concat(titles, ";")

end


function p.main(frame)

    -- bringing in property values from the appropriate set for this item
    local pData = mw.loadData('Module:GetData/'..frame.args['data'])

    -- bringing in the arguments passed to the parent templates
    local parentArgs = frame:getParent().args

    -- bringing in the properties being requested and feeding into a table
    local props = parentArgs.prop
    local propTable = {}
    for propValue in string.gmatch(props, "([^;]+)") do
        table.insert(propTable, propValue)
    end

    -- checking if a single property called actually has a value, if not return nothing
    if #propTable == 1 then
        local valueCheck = propTable[1]
        if pData[valueCheck] == '' or pData[valueCheck] == nil then 
            if parentArgs.default then
                return parentArgs.default
            else
                return ''
            end
        end
    end

    -- defining our output variable
    local finalString = ""

    -- if an intro string has been defined, adding to output string now
    if parentArgs.intro then
        for k,v in ipairs(propTable) do
            if pData[v] ~= "" then
                finalString = parentArgs.intro
            end
        end
    end

    -- iterating through requested properties
    for i,prop in pairs(propTable) do
        if pData[prop] then

            -- feeding property value into a variable
            local pdString = pData[prop]

            -- checing if a "map" parameter has been defined
            if parentArgs.map then
                local pdTable = {}

                -- checking if a sep parameter has been defined and splitting values accordingly
                if parentArgs.sep then
                    local sep = parentArgs.sep
                    for value in string.gmatch(pdString, "([^" .. sep .. "]+)") do
                        table.insert(pdTable, value)
                    end
                else
                    table.insert(pdTable, pdString)
                end

                -- mapping each value from above
                local map = parentArgs.map
                pdString = ""
                for k,v in pairs(pdTable) do
                    if v and tostring(v) then
                        mapdVal = string.gsub(map, "@@@", v)
                        pdString = pdString .. mapdVal
                    end
                end
            end

            -- checking if a printsep parameter has been defined and separating each mapped value accordingly
            if parentArgs.printsep then
                finalString = finalString .. parentArgs.printsep .. frame:preprocess( pdString )
                if mw.ustring.sub(finalString, 1, 1) == parentArgs.printsep then
                    finalString = mw.ustring.sub(finalString, 2)
                end
            else
                finalString = finalString .. frame:preprocess( pdString )
            end
        end
    end

    -- if an outro string has been defined, adding to output string now
    if parentArgs.outro then
        for k,v in ipairs(propTable) do
            if pData[v] ~= "" then
                finalString = finalString .. parentArgs.outro
            end
        end
    end

    -- checking if a silent flag has been added, if so and string is empty anyways, return string, otherwise, check for default parent argument and return that.
    if parentArgs.silent ~= "yes" then
        if finalString ~= "" and finalString ~= nil then
            return finalString
        else
            if parentArgs.default then
                return parentArgs.default
            end
        end
    end
end

return p