Coverage for app/main.py: 96%

1#!/usr/bin/env python3

2# -*- coding: utf-8; mode: python -*-

4# This program is free software; you can redistribute it and/or modify

5# it under the terms of the GNU General Public License as published by

6# the Free Software Foundation; either version 2 of the License, or

7# (at your option) any later version.

9# This program is distributed in the hope that it will be useful,

10# but WITHOUT ANY WARRANTY; without even the implied warranty of

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12# GNU General Public License for more details.

14# You should have received a copy of the GNU General Public License along

15# with this program; if not, write to the Free Software Foundation, Inc.,

16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

18"""A Python CGI webservice to provide marketplace functionality for

19TextGridLab/Eclipse.

21Provides the necessary XML files for showing content in the

22Marketplace Menu. Content itself comes from two sources: an XML

23configuration file next to this script and another system which gives

24information about the plugin to the users and to this webservice.

26In this specific case, this is the Atlassian Confluence Wiki system

27which also provides their content via REST API:

28https://dev2.dariah.eu/wiki/rest/prototype/1/content/27329537

30A fourth component is a .htaccess file which deals with the rewriting

31of URLs to cater for all the needs. Using htaccess means of course

32that we need an Apache webserver.

34The system also expects an image file to be used as a logo.

36To try out the functionality, the ini file of the Lab has to be tweaked:

37-Dorg.eclipse.epp.internal.mpc.core.service.DefaultCatalogService.url=http://ocropus.rz-berlin.mpg.de/~kthoden/m/

38Or point the URL to your own private instance.

40Query by Eclipse looks like this:

41http://ocropus.rz-berlin.mpg.de/~kthoden/m/featured/6/api/p?product=info.textgrid.lab.core.application.base_product&os=macosx&runtime.version=3.7.0.v20110110&client=org.eclipse.epp.mpc.core&java.version=1.6.0_65&product.version=0.0.2.201310011243&ws=cocoa&nl=de_DE

43The reference of the Eclipse interface is at http://wiki.eclipse.org/Marketplace/REST

45Some new stuff:

46- first step: a plugin has to be registered at page 36342854

47- we get the list from there

49"""

50__author__ = "Klaus Thoden, kthoden@mpiwg-berlin.mpg.de"

51__date__ = "2017-05-19"

53###########

54# Imports #

55###########

56import os

57import logging

58import yaml

59import requests

60from configparser import ConfigParser

61from lxml import etree

63from fastapi import FastAPI, Path, Response, HTTPException

64from fastapi.responses import PlainTextResponse, HTMLResponse

65from fastapi.exceptions import RequestValidationError

66from starlette.exceptions import HTTPException as StarletteHTTPException

68# setting up things

69# both config and cache directory are in the same place as this script

70CONFIG = ConfigParser()

71CONFIG.read('etc/default.conf')

73# override config from the general section from environment, for configuration inside docker.

74# environment variables starting with MS_GENERAL_ are mapped, example:

75# MS_GENERAL_LOGFILE -> CONFIG['General']['logfile']

76env_vars=os.environ

77for key in env_vars:

78 if(key.startswith("MS_GENERAL_")):

79 confkey=key[11:].lower()

80 confval=env_vars[key]

81 CONFIG.set("General", confkey, confval)

83LOGFILE = CONFIG['General']['logfile']

84LOGLEVEL = CONFIG['General']['loglevel']

86numeric_level = getattr(logging, LOGLEVEL.upper(), None)

87if not isinstance(numeric_level, int):

88 raise ValueError('Invalid log level: %s' % loglevel)

89logging.basicConfig(filename=LOGFILE, level=numeric_level, format='%(asctime)s - %(levelname)s - %(message)s')

91WIKI_VIEW = CONFIG['General']['wiki_view']

93# adaptable XPath for info that is parsed out of the confluence page

94# was /pluginInfo/table/

95PLUGIN_INFO_TABLE_XPATH = "/pluginInfo//table[1]/"

97HEADERLINE = 'Content-Type: text/%s; charset=utf-8\n'

99###########

100# Objects #

101###########

102class TGLab():

103 """Class for storing information about the Lab that sent the query.

104 # We should maybe catch and retain all those things the client tells us:

105 # product=info.textgrid.lab.core.application.base_product

106 # os=macosx

107 # runtime.version=3.7.0.v20110110

108 # client=org.eclipse.epp.mpc.core

109 # java.version=1.6.0_65

110 # product.version=0.0.2.201310011243

111 # ws=cocoa

112 # nl=de_DE

113"""

114 pass

115# class TGLab ends here

116

117class PlugIn():

118 """Class for Plugins, just to collect their properties. Has one

119 required positional argument, the confluence pageId."""

120 def __init__(self,

121 pageId,

122 name = "",

123 human_title = "",

124 description = "",

125 featured = False,

126 logo = "",

127 license = "",

128 plugId = "",

129 category = "",

130 installableUnit = "",

131 screenshot = "",

132 owner = CONFIG['General']['company'],

133 company = CONFIG['General']['company'],

134 company_url = CONFIG['General']['company_url'],

135 update_url = CONFIG['General']['update_url']):

136 self.human_title = human_title

137 self.description = description

138 self.logo = logo

139 self.license = license

140 self.plugId = str(plugId)

141 self.featured = featured

142 self.name = name

143 self.category = str(category)

144 self.pageId = str(pageId)

145 self.screenshot = screenshot

146 self.installableUnit = installableUnit

147 self.owner = owner

148 self.company = company

149 self.company_url = company_url

150 self.update_url = update_url

151# class PlugIn ends here

152

153class MarketPlace():

154 """Why not have that, too"""

155 def __init__(self, human_title, desc, mpid, name, url, icon, company, company_url, update_url, main_wiki_page):

156 self.human_title = human_title

157 self.desc = desc

158 self.mpid = mpid

159 self.name = name

160 self.url = url

161 self.icon = icon

162 self.company = company

163 self.company_url = company_url

164 self.update_url = update_url

165 self.main_wiki_page = main_wiki_page

166# class MarketPlace ends here

167

168# create Marketplace object

169MPLACE = MarketPlace(

170 CONFIG['General']['human_title'],

171 CONFIG['General']['description'],

172 CONFIG['General']['id'],

173 CONFIG['General']['name'],

174 CONFIG['General']['url'],

175 CONFIG['General']['icon'],

176 CONFIG['General']['company'],

177 CONFIG['General']['company_url'],

178 CONFIG['General']['update_url'],

179 CONFIG['General']['main_wiki_page'])

180

181################

182# YAML parsing #

183################

184def plugin_constructor(loader, node):

185 fields = loader.construct_mapping(node)

186 return PlugIn(**fields)

187

188yaml.SafeLoader.add_constructor('!PlugIn', plugin_constructor)

189

190def load_data():

191 with open('etc/data.yaml', 'r', encoding='utf-8') as stream:

192 PLUGINS = yaml.safe_load(stream)

193 return PLUGINS

194

195#############################################

196# Here starts the building of the XML nodes #

197#############################################

198def build_mp_apip():

199 """Return info about the whole marketplace. Which categories are in there?"""

200

201 # building the XML

202 mplace = etree.Element("marketplace")

203 market = etree.SubElement(mplace, "market",

204 id=MPLACE.mpid,

205 name=MPLACE.name,

206 url=MPLACE.url + "/category/markets/" + MPLACE.mpid)

207

208 categ = list(CONFIG['Categories'].values())

209 cat_id = list(CONFIG['Categories'].keys())

210

211 # Iterating through the categories

212 cat_count = 1

213 for cat_key, cat_val in zip(categ, cat_id):

214 etree.SubElement(market, "category",

215 count=str(cat_count),

216 id=cat_val,

217 name=cat_key,

218 url=str(MPLACE.url) + "/taxonomy/term/" + MPLACE.mpid + "," + cat_key)

219 # is the space after mpid+","+cat_key) obligatory???

220 # url=str(MPLACE.url) + "/taxonomy/term/" + MPLACE.mpid + ", " + cat_key)

221 cat_count += 1

222 return mplace

223# def build_mp_apip ends here

224

225def build_mp_cat_apip():

226 """Return information on a catalog. According to server log, this is

227 the first thing the Lab looks for. Requires only info from config

228 file. After choosing that catalog, the root is called.

229 """

230 # build the XML

231 mplace = etree.Element("marketplace")

232 catalogs = etree.SubElement(mplace, "catalogs")

233 catalog = etree.SubElement(catalogs, "catalog",

234 id=MPLACE.mpid,

235 title=MPLACE.human_title,

236 url=MPLACE.url,

237 selfContained="1",

238 icon=MPLACE.url + "/" + MPLACE.icon)

239 desc = etree.SubElement(catalog, "description").text = MPLACE.desc

240 dep_rep = etree.SubElement(catalog, "dependenciesRepository")

241 wizard = etree.SubElement(catalog, "wizard", title="")

242 icon = etree.SubElement(wizard, "icon")

243

244 if CONFIG['General']['search'] != "0":

245 search_tab = etree.SubElement(wizard, "searchtab", enabled="1").text = "Suche"

246 else:

247 search_tab = etree.SubElement(wizard, "searchtab", enabled="0").text = "Suche"

248 if CONFIG['General']['popular'] != "0":

249 pop_tab = etree.SubElement(wizard, "populartab", enabled="1").text = "Beliebt"

250 else:

251 pop_tab = etree.SubElement(wizard, "populartab", enabled="0").text = "Beliebt"

252 if CONFIG['General']['recent'] != "0":

253 rec_tab = etree.SubElement(wizard, "recenttab", enabled="1").text = "Neu"

254 else:

255 rec_tab = etree.SubElement(wizard, "recenttab", enabled="0").text = "Neu"

256

257 return mplace

258# def build_mp_cat_apip ends here

259

260def build_mp_taxonomy(market_id, cate_id, PLUGINS):

261 """Construct the taxonomy. List all plugins of one category. The

262 category a plugin belongs to is taken from the config."""

263

264 # small dictionary for handling the category name and id

265 cate_dict = {}

266 for cat_key, cat_val in zip(list(CONFIG['Categories'].values()), list(CONFIG['Categories'].keys())):

267 cate_dict.update({cat_val:cat_key})

268

269 # a small detour, because we might get the name value of the category instead of the Id

270 if cate_id in [v for k, v in list(cate_dict.items())]:

271 cate_id = [k for k, v in list(cate_dict.items()) if v == cate_id][0]

272

273 # build the XML

274 mplace = etree.Element("marketplace")

275 category = etree.SubElement(mplace, "category",

276 id=str(cate_id),

277 name=(cate_dict[cate_id]),

278 url=MPLACE.url + "/taxonomy/term/" + str(market_id) + "," + str(cate_id))

279 # is the space after mpid+","+cat_key) obligatory???

280 # url=MPLACE.url + "/taxonomy/term/" + str(market_id) + ", " + str(cate_id))

281

282 # repeat for those belonging to the same group

283 for iu in PLUGINS:

284 if int(iu.category) == int(cate_id):

285 node = etree.SubElement(category, "node",

286 id = iu.plugId,

287 name = iu.human_title,

288 url = MPLACE.url + "/content/" + iu.plugId)

289 # do something about this!!

290 fav = etree.SubElement(category, "favorited").text = "0"

291

292 return mplace

293# def build_mp_taxonomy ends here

294

295def build_mp_node_apip(plug_id, PLUGINS):

296 """Return info on installable Unit (i.e. plugin). Get info from the

297 CONFIG and from Confluence info page. Input is plug_id, identifier

298 of the plugin

299 """

300

301 # find out which Plugin we need

302 for candidate in PLUGINS:

303 if candidate.plugId == plug_id:

304 current_plugin = candidate

305

306 node = etree.Element("node",

307 id = current_plugin.plugId,

308 name = current_plugin.human_title,

309 url = MPLACE.url + "/content/" + current_plugin.plugId)

310

311 body_element = etree.SubElement(node, "body").text = etree.CDATA(current_plugin.description)

312 # taken from Label of wikipage

313 cate_element = etree.SubElement(node, "categories")

314 # noch nicht ganz fertig!

315 category = etree.SubElement(cate_element, "categories",

316 id = current_plugin.category,

317 name = current_plugin.human_title,

318 url = MPLACE.url + "/taxonomy/term/" + MPLACE.mpid + "," + current_plugin.category)

319 # how to do that?

320 change_element = etree.SubElement(node, "changed").text = "0"

321 # constantly TextGrid? can be superseded by plugin-specific entry

322 company_element = etree.SubElement(node, "companyname").text = etree.CDATA(current_plugin.company)

323 # upload of plugin?, use old values here?

324 created_element = etree.SubElement(node, "created").text = "0"

325 # what here?

326 eclipse_element = etree.SubElement(node, "eclipseversion").text = etree.CDATA("0")

327 # would that be ticked on the wiki page?

328 fav_element = etree.SubElement(node, "favorited").text = "0"

329 # 1 is original value here

330 foundation_element = etree.SubElement(node, "foundationmember").text = "1"

331 url_element = etree.SubElement(node, "homepageurl").text = etree.CDATA(current_plugin.company_url)

332 # icon of plugin

333 if current_plugin.logo.startswith('http'):

334 image_element = etree.SubElement(node, "image").text = etree.CDATA(current_plugin.logo)

335 else:

336 image_element = etree.SubElement(node, "image").text = etree.CDATA("https://dev2.dariah.eu/wiki/download/attachments/" + current_plugin.pageId + "/" + current_plugin.logo)

337

338 # just a container

339 ius_element = etree.SubElement(node, "ius")

340 iu_element = etree.SubElement(ius_element, "iu").text = current_plugin.installableUnit

341 license_element = etree.SubElement(node, "license").text = current_plugin.license

342 # who is the owner? same as company!

343 owner_element = etree.SubElement(node, "owner").text = etree.CDATA(current_plugin.owner)

344 # what is this about?

345 resource_element = etree.SubElement(node, "resource")

346 # see logo

347 # screenshot would be displayed if we click on more info in marketplace

348 if len(current_plugin.screenshot) != 0:

349 scrshotEle = etree.SubElement(node, "screenshot").text = etree.CDATA("https://dev2.dariah.eu/wiki/download/attachments/" + current_plugin.pageId + "/" + current_plugin.screenshot)

350 # also hidden field?

351 update_element = etree.SubElement(node, "updateurl").text = etree.CDATA(current_plugin.update_url)

352 return node

353# def build_mp_node_apip ends here

354

355def build_mp_frfp_apip(list_type, PLUGINS, mark_id=CONFIG['General']['id']):

356 """Take those nodes (my theory here) that have a value of non-nil in

357 'featured' (should be on the wiki page) and wraps them into some

358 XML. Works also for recent, favorite and popular, they are

359 similar. Hence the name of this function.

360

361 This one needs to be fleshed out!

362 """

363

364 # the heart of everything. This list contains the plugins to be displayed!

365 # controlled by the configuration page in the wiki.

366 featured_list = []

367

368 # find out which Plugin we need

369 # for now, just display all

370 for candidate in PLUGINS:

371 featured_list.append(candidate.plugId)

372 # if candidate.featured != "N":

373 # # featured_list.append(candidate.plugId)

374

375

376 mplace = etree.Element("marketplace")

377 plugin_list = etree.SubElement(mplace, list_type, count=str(len(featured_list)))

378 # make the nodes here as a subElement of the list

379 for item in featured_list:

380 new_node = build_mp_node_apip(item, PLUGINS)

381 plugin_list.insert(1, new_node)

382

383 return mplace

384# def build_mp_frfp_apip ends here

385

386def build_mp_content_apip(plug_id, PLUGINS):

387 """Return info on a single node. The node_id is """

388

389 mplace = etree.Element("marketplace")

390 new_node = build_mp_node_apip(plug_id, PLUGINS)

391 mplace.insert(1, new_node)

392

393 return mplace

394# def build_mp_content_apip ends here

395

396##########

397# Output #

398##########

399def xmlresponse(node):

400 xml = etree.tostring(node, pretty_print=True, encoding='utf-8', xml_declaration=True)

401 return Response(content=xml, media_type='application/xml')

402

403

404##########

405# routes #

406##########

407app = FastAPI(

408 title="TextGridLab Marketplace",

409 description="This is the API for the TextGridLab Marketplace, an implementation of the [Eclipse Marketplace API](https://wiki.eclipse.org/Marketplace/REST)",

410 version="2.0.0",

411 docs_url='/marketplace/docs',

412 redoc_url='/marketplace/redoc',

413 openapi_url='/marketplace/openapi.json'

414)

415

416# define xml response content type for openapi

417xmlresponsedef = {

418 200: {

419 "content": {

420 "application/xml": {}

421 }

422 },

423 422: {

424 "description": "Validation Error",

425 "content": {

426 "text/plain": {}

427 }

428 }

429}

430

431@app.get("/marketplace/api/p",

432 summary="List Markets and Categories",

433 description="""This will return a listing of Markets and Categories, it includes URLs for each category, as well number of listings in each category.

434 See [Retrieving A listing of Markets and Categories](https://web.archive.org/web/20200220202907/https://wiki.eclipse.org/Marketplace/REST#Retrieving_A_listing_of_Markets_and_Categories)""",

435 response_class=Response,

436 responses=xmlresponsedef

437)

438def main_api_p():

439 node = build_mp_apip()

440 return xmlresponse(node)

441

442

443@app.get("/marketplace/catalogs/api/p",

444 summary="List all Catalogs",

445 description="""This will return a listing of all catalogs that are browsable with the MPC. It also includes basic branding parameters,

446 like title and icon and strategies resolving dependencies.

447 See [Retrieving a listing of all catalogs](https://web.archive.org/web/20200220202907/https://wiki.eclipse.org/Marketplace/REST#Retrieving_a_listing_of_all_catalogs)""",

448 response_class=Response,

449 responses=xmlresponsedef

450)

451def catalogs_api_p():

452 node = build_mp_cat_apip()

453 return xmlresponse(node)

454

455

456@app.get("/marketplace/taxonomy/term/{market_id},{category_id}/api/p",

457 summary="Listings from a specific Market / Category",

458 response_class=Response,

459 responses=xmlresponsedef)