alkimia  8.0.2
alkonlinequote.cpp
Go to the documentation of this file.
1 /***************************************************************************
2  * Copyright 2004 Ace Jones <acejones@users.sourceforge.net> *
3  * *
4  * This file is part of libalkimia. *
5  * *
6  * libalkimia is free software; you can redistribute it and/or *
7  * modify it under the terms of the GNU General Public License *
8  * as published by the Free Software Foundation; either version 2.1 of *
9  * the License or (at your option) version 3 or any later version. *
10  * *
11  * libalkimia is distributed in the hope that it will be useful, *
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of *
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
14  * GNU General Public License for more details. *
15  * *
16  * You should have received a copy of the GNU General Public License *
17  * along with this program. If not, see <http://www.gnu.org/licenses/> *
18  ***************************************************************************/
19 
20 #include "alkonlinequote.h"
21 
22 #include "alkdateformat.h"
23 #include "alkexception.h"
24 #include "alkfinancequoteprocess.h"
25 #include "alkonlinequoteprocess.h"
26 #include "alkonlinequotesprofile.h"
28 #include "alkonlinequotesource.h"
29 #include "alkwebpage.h"
30 
31 #include <QApplication>
32 #include <QByteArray>
33 #include <QFile>
34 //#include <QFileInfo>
35 #include <QRegExp>
36 #include <QTextStream>
37 #include <QTextCodec>
38 
39 #include <KConfigGroup>
40 #include <KDebug>
41 #include <KEncodingProber>
42 #include <KGlobal>
43 #include <kio/netaccess.h>
44 #include <kio/scheduler.h>
45 #include <KLocale>
46 #include <KProcess>
47 #include <KShell>
48 #include <KUrl>
49 
51 {
52 }
53 
55 {
56  m_type.append(type);
57 }
58 
60 {
61  m_type = e.m_type;
62 }
63 
65 {
66  if (!m_type.contains(t)) {
67  m_type.append(t);
68  }
69  return *this;
70 }
71 
73 {
74  return m_type.contains(t);
75 }
76 
77 
78 class AlkOnlineQuote::Private : public QObject
79 {
80  Q_OBJECT
81 public:
84  QString m_quoteData;
85  QString m_symbol;
86  QString m_id;
87  QDate m_date;
88  double m_price;
91  KUrl m_url;
92  QEventLoop *m_eventLoop;
96 
97  static int dbgArea()
98  {
99  static int s_area = KDebug::registerArea("Alkimia (AlkOnlineQuote)");
100  return s_area;
101  }
102 
104  : m_p(parent)
105  , m_eventLoop(nullptr)
106  , m_ownProfile(false)
107  {
108  connect(&m_filter, SIGNAL(processExited(QString)), this, SLOT(slotParseQuote(QString)));
109  }
110 
112  {
113  if (m_ownProfile)
114  delete m_profile;
115  }
116 
117  bool initLaunch(const QString &_symbol, const QString &_id, const QString &_source);
118  bool launchWebKitCssSelector(const QString &_symbol, const QString &_id,
119  const QString &_source);
120  bool launchWebKitHtmlParser(const QString &_symbol, const QString &_id, const QString &_source);
121  bool launchNative(const QString &_symbol, const QString &_id, const QString &_source);
122  bool launchFinanceQuote(const QString &_symbol, const QString &_id, const QString &_source);
123  void enter_loop();
124  bool parsePrice(const QString &pricestr);
125  bool parseDate(const QString &datestr);
126 
127 public slots:
128  void slotLoadStarted();
129  void slotLoadFinishedHtmlParser(bool ok);
130  void slotLoadFinishedCssSelector(bool ok);
131  bool slotParseQuote(const QString &_quotedata);
132 };
133 
134 bool AlkOnlineQuote::Private::initLaunch(const QString &_symbol, const QString &_id, const QString &_source)
135 {
136  m_symbol = _symbol;
137  m_id = _id;
138  m_errors = Errors::None;
139 
140  emit m_p->status(QString("(Debug) symbol=%1 id=%2...").arg(_symbol, _id));
141 
142  // Get sources from the config file
143  QString source = _source;
144  if (source.isEmpty()) {
145  source = "KMyMoney Currency";
146  }
147 
148  if (!m_profile->quoteSources().contains(source)) {
149  emit m_p->error(i18n("Source <placeholder>%1</placeholder> does not exist.", source));
150  m_errors |= Errors::Source;
151  return false;
152  }
153 
154  //m_profile->createSource(source);
155  m_source = AlkOnlineQuoteSource(source, m_profile);
156 
157  KUrl url;
158 
159  // if the source has room for TWO symbols..
160  if (m_source.url().contains("%2")) {
161  // this is a two-symbol quote. split the symbol into two. valid symbol
162  // characters are: 0-9, A-Z and the dot. anything else is a separator
163  QRegExp splitrx("([0-9a-z\\.]+)[^a-z0-9]+([0-9a-z\\.]+)", Qt::CaseInsensitive);
164  // if we've truly found 2 symbols delimited this way...
165  if (splitrx.indexIn(m_symbol) != -1) {
166  url = KUrl(m_source.url().arg(splitrx.cap(1), splitrx.cap(2)));
167  } else {
168  kDebug(Private::dbgArea()) << "WebPriceQuote::launch() did not find 2 symbols";
169  }
170  } else {
171  // a regular one-symbol quote
172  url = KUrl(m_source.url().arg(m_symbol));
173  }
174 
175  m_url = url;
176 
177  return true;
178 }
179 
181 {
182  if (!ok) {
183  emit m_p->error(i18n("Unable to fetch url for %1").arg(m_symbol));
184  m_errors |= Errors::URL;
185  emit m_p->failed(m_id, m_symbol);
186  } else {
187  // parse symbol
188  slotParseQuote(AlkOnlineQuotesProfileManager::instance().webPage()->toHtml());
189  }
190  if (m_eventLoop)
191  m_eventLoop->exit();
192 }
193 
195 {
196  if (!ok) {
197  emit m_p->error(i18n("Unable to fetch url for %1").arg(m_symbol));
198  m_errors |= Errors::URL;
199  emit m_p->failed(m_id, m_symbol);
200  } else {
202  // parse symbol
203  QString symbol = webPage->getFirstElement(m_source.sym());
204  if (!symbol.isEmpty()) {
205  emit m_p->status(i18n("Symbol found: '%1'", symbol));
206  } else {
207  m_errors |= Errors::Symbol;
208  emit m_p->error(i18n("Unable to parse symbol for %1", m_symbol));
209  }
210 
211  // parse price
212  QString price = webPage->getFirstElement(m_source.price());
213  bool gotprice = parsePrice(price);
214 
215  // parse date
216  QString date = webPage->getFirstElement(m_source.date());
217  bool gotdate = parseDate(date);
218 
219  if (gotprice && gotdate) {
220  emit m_p->quote(m_id, m_symbol, m_date, m_price);
221  } else {
222  emit m_p->failed(m_id, m_symbol);
223  }
224  }
225  if (m_eventLoop)
226  m_eventLoop->exit();
227 }
228 
230 {
231  emit m_p->status(i18n("Fetching URL %1...", m_url.prettyUrl()));
232 }
233 
234 bool AlkOnlineQuote::Private::launchWebKitCssSelector(const QString &_symbol, const QString &_id,
235  const QString &_source)
236 {
237  if (!initLaunch(_symbol, _id, _source)) {
238  return false;
239  }
241  connect(webPage, SIGNAL(loadStarted()), this, SLOT(slotLoadStarted()));
242  connect(webPage, SIGNAL(loadFinished(bool)), this,
243  SLOT(slotLoadFinishedCssSelector(bool)));
244  webPage->setUrl(m_url);
245  m_eventLoop = new QEventLoop;
246  m_eventLoop->exec();
247  delete m_eventLoop;
248  disconnect(webPage, SIGNAL(loadStarted()), this, SLOT(slotLoadStarted()));
249  disconnect(webPage, SIGNAL(loadFinished(bool)), this,
250  SLOT(slotLoadFinishedCssSelector(bool)));
251 
252  return !(m_errors & Errors::URL || m_errors & Errors::Price
253  || m_errors & Errors::Date || m_errors & Errors::Data);
254 }
255 
256 bool AlkOnlineQuote::Private::launchWebKitHtmlParser(const QString &_symbol, const QString &_id,
257  const QString &_source)
258 {
259  if (!initLaunch(_symbol, _id, _source)) {
260  return false;
261  }
263  connect(webPage, SIGNAL(loadStarted()), this, SLOT(slotLoadStarted()));
264  connect(webPage, SIGNAL(loadFinished(bool)), this, SLOT(slotLoadFinishedHtmlParser(bool)));
265  webPage->load(m_url, m_acceptLanguage);
266  m_eventLoop = new QEventLoop;
267  m_eventLoop->exec();
268  delete m_eventLoop;
269  disconnect(webPage, SIGNAL(loadStarted()), this, SLOT(slotLoadStarted()));
270  disconnect(webPage, SIGNAL(loadFinished(bool)), this, SLOT(slotLoadFinishedHtmlParser(bool)));
271 
272  return !(m_errors & Errors::URL || m_errors & Errors::Price
273  || m_errors & Errors::Date || m_errors & Errors::Data);
274 }
275 
276 bool AlkOnlineQuote::Private::launchNative(const QString &_symbol, const QString &_id,
277  const QString &_source)
278 {
279  bool result = true;
280  if (!initLaunch(_symbol, _id, _source)) {
281  return false;
282  }
283 
284  KUrl url = m_url;
285  if (url.isLocalFile()) {
286  emit m_p->status(i18nc("The process x is executing", "Executing %1...", url.toLocalFile()));
287 
288  m_filter.clearProgram();
289  m_filter << url.toLocalFile().split(' ', QString::SkipEmptyParts);
290  m_filter.setSymbol(m_symbol);
291 
292  m_filter.setOutputChannelMode(KProcess::MergedChannels);
293  m_filter.start();
294 
295  if (m_filter.waitForStarted()) {
296  result = true;
297  } else {
298  emit m_p->error(i18n("Unable to launch: %1", url.toLocalFile()));
299  m_errors |= Errors::Script;
300  result = slotParseQuote(QString());
301  }
302  } else {
303  slotLoadStarted();
304 
305  QString tmpFile;
306  if (KIO::NetAccess::download(url, tmpFile, 0)) {
307  // kDebug(Private::dbgArea()) << "Downloaded " << tmpFile;
308  kDebug(Private::dbgArea()) << "Downloaded" << tmpFile << "from" << url;
309  QFile f(tmpFile);
310  if (f.open(QIODevice::ReadOnly)) {
311  // Find out the page encoding and convert it to unicode
312  QByteArray page = f.readAll();
313  KEncodingProber prober(KEncodingProber::Universal);
314  prober.feed(page);
315  QTextCodec *codec = QTextCodec::codecForName(prober.encoding());
316  if (!codec) {
317  codec = QTextCodec::codecForLocale();
318  }
319  QString quote = codec->toUnicode(page);
320  f.close();
321  emit m_p->status(i18n("URL found: %1...", url.prettyUrl()));
323  AlkOnlineQuotesProfileManager::instance().webPage()->setContent(quote.toLocal8Bit());
324  result = slotParseQuote(quote);
325  } else {
326  emit m_p->error(i18n("Failed to open downloaded file"));
327  m_errors |= Errors::URL;
328  result = slotParseQuote(QString());
329  }
330  KIO::NetAccess::removeTempFile(tmpFile);
331  } else {
332  emit m_p->error(KIO::NetAccess::lastErrorString());
333  m_errors |= Errors::URL;
334  result = slotParseQuote(QString());
335  }
336  }
337  return result;
338 }
339 
340 bool AlkOnlineQuote::Private::launchFinanceQuote(const QString &_symbol, const QString &_id,
341  const QString &_sourcename)
342 {
343  bool result = true;
344  m_symbol = _symbol;
345  m_id = _id;
346  m_errors = Errors::None;
347 #if 0
348  QString FQSource = _sourcename.section(' ', 1);
349  m_source = AlkOnlineQuoteSource(_sourcename, m_financeQuoteScriptPath,
350  "\"([^,\"]*)\",.*", // symbol regexp
351  "[^,]*,[^,]*,\"([^\"]*)\"", // price regexp
352  "[^,]*,([^,]*),.*", // date regexp
353  "%y-%m-%d"); // date format
354 
355  //emit status(QString("(Debug) symbol=%1 id=%2...").arg(_symbol,_id));
356 
357  m_filter.clearProgram();
358  m_filter << "perl" << m_financeQuoteScriptPath << FQSource << KShell::quoteArg(_symbol);
359  m_filter.setSymbol(m_symbol);
360  emit m_p->status(i18nc("Executing 'script' 'online source' 'investment symbol' ",
361  "Executing %1 %2 %3...", m_financeQuoteScriptPath, FQSource, _symbol));
362 
363  m_filter.setOutputChannelMode(KProcess::MergedChannels);
364  m_filter.start();
365 
366  // This seems to work best if we just block until done.
367  if (m_filter.waitForFinished()) {
368  } else {
369  emit m_p->error(i18n("Unable to launch: %1", m_financeQuoteScriptPath));
370  m_errors |= Errors::Script;
371  result = slotParseQuote(QString());
372  }
373 #else
374  Q_UNUSED(_sourcename);
375 #if !defined(Q_CC_MSVC)
376  #warning to be implemented
377 #endif
378 #endif
379 
380  return result;
381 }
382 
383 bool AlkOnlineQuote::Private::parsePrice(const QString &_pricestr)
384 {
385  bool result = true;
386  // Deal with european quotes that come back as X.XXX,XX or XX,XXX
387  //
388  // We will make the assumption that ALL prices have a decimal separator.
389  // So "1,000" always means 1.0, not 1000.0.
390  //
391  // Remove all non-digits from the price string except the last one, and
392  // set the last one to a period.
393  QString pricestr(_pricestr);
394  if (!pricestr.isEmpty()) {
395  int pos = pricestr.lastIndexOf(QRegExp("\\D"));
396  if (pos > 0) {
397  pricestr[pos] = '.';
398  pos = pricestr.lastIndexOf(QRegExp("\\D"), pos - 1);
399  }
400  while (pos > 0) {
401  pricestr.remove(pos, 1);
402  pos = pricestr.lastIndexOf(QRegExp("\\D"), pos);
403  }
404 
405  m_price = pricestr.toDouble();
406  kDebug(Private::dbgArea()) << "Price" << pricestr;
407  emit m_p->status(i18n("Price found: '%1' (%2)", pricestr, m_price));
408  } else {
409  m_errors |= Errors::Price;
410  emit m_p->error(i18n("Unable to parse price for '%1'", m_symbol));
411  result = false;
412  }
413  return result;
414 }
415 
416 bool AlkOnlineQuote::Private::parseDate(const QString &datestr)
417 {
418  if (!datestr.isEmpty()) {
419  emit m_p->status(i18n("Date found: '%1'", datestr));
420 
421  AlkDateFormat dateparse(m_source.dateformat());
422  try {
423  m_date = dateparse.convertString(datestr, false /*strict*/);
424  kDebug(Private::dbgArea()) << "Date" << datestr;
425  emit m_p->status(i18n("Date format found: '%1' -> '%2'", datestr, m_date.toString()));
426  } catch (const AlkException &e) {
427  m_errors |= Errors::DateFormat;
428  emit m_p->error(i18n("Unable to parse date '%1' using format '%2': %3").arg(datestr,
429  dateparse.format(),
430  e.what()));
431  m_date = QDate::currentDate();
432  emit m_p->status(i18n("Using current date for '%1'").arg(m_symbol));
433  }
434  } else {
435  m_errors |= Errors::Date;
436  emit m_p->error(i18n("Unable to parse date for '%1'").arg(m_symbol));
437  m_date = QDate::currentDate();
438  emit m_p->status(i18n("Using current date for '%1'").arg(m_symbol));
439  }
440  return true;
441 }
442 
450 bool AlkOnlineQuote::Private::slotParseQuote(const QString &_quotedata)
451 {
452  QString quotedata = _quotedata;
453  m_quoteData = quotedata;
454  bool gotprice = false;
455  bool gotdate = false;
456  bool result = true;
457 
458  kDebug(Private::dbgArea()) << "quotedata" << _quotedata;
459 
460  if (!quotedata.isEmpty()) {
461  if (!m_source.skipStripping()) {
462  //
463  // First, remove extraneous non-data elements
464  //
465 
466  // HTML tags
467  quotedata.remove(QRegExp("<[^>]*>"));
468 
469  // &...;'s
470  quotedata.replace(QRegExp("&\\w+;"), " ");
471 
472  // Extra white space
473  quotedata = quotedata.simplified();
474  kDebug(Private::dbgArea()) << "stripped text" << quotedata;
475  }
476 
477  QRegExp symbolRegExp(m_source.sym());
478  QRegExp dateRegExp(m_source.date());
479  QRegExp priceRegExp(m_source.price());
480 
481  if (symbolRegExp.indexIn(quotedata) > -1) {
482  kDebug(Private::dbgArea()) << "Symbol" << symbolRegExp.cap(1);
483  emit m_p->status(i18n("Symbol found: '%1'", symbolRegExp.cap(1)));
484  } else {
485  m_errors |= Errors::Symbol;
486  emit m_p->error(i18n("Unable to parse symbol for %1", m_symbol));
487  }
488 
489  if (priceRegExp.indexIn(quotedata) > -1) {
490  gotprice = true;
491  QString pricestr = priceRegExp.cap(1);
492  parsePrice(pricestr);
493  } else {
494  parsePrice(QString());
495  }
496 
497  if (dateRegExp.indexIn(quotedata) > -1) {
498  gotdate = true;
499  QString datestr = dateRegExp.cap(1);
500  parseDate(datestr);
501  } else {
502  parseDate(QString());
503  }
504 
505  if (gotprice && gotdate) {
506  emit m_p->quote(m_id, m_symbol, m_date, m_price);
507  } else {
508  emit m_p->failed(m_id, m_symbol);
509  result = false;
510  }
511  } else {
512  m_errors |= Errors::Data;
513  emit m_p->error(i18n("Unable to update price for %1 (empty quote data)", m_symbol));
514  emit m_p->failed(m_id, m_symbol);
515  result = false;
516  }
517  return result;
518 }
519 
521  : QObject(_parent)
522  , d(new Private(this))
523 {
524  if (profile)
525  d->m_profile = profile;
526  else {
528  d->m_ownProfile = true;
529  }
530 }
531 
533 {
534  delete d;
535 }
536 
538 {
539  return d->m_profile;
540 }
541 
543 {
544  if (profile && d->m_ownProfile) {
545  // switching from own profile to external
546  delete d->m_profile;
547  d->m_ownProfile = false;
548  d->m_profile = profile;
549 
550  } else if (!profile && !d->m_ownProfile) {
551  // switching from external to own profile
553  d->m_ownProfile = true;
554 
555  } else if (profile) {
556  // exchange external profile
557  d->m_profile = profile;
558  }
559 }
560 
561 void AlkOnlineQuote::setAcceptLanguage(const QString &language)
562 {
563  d->m_acceptLanguage = language;
564 }
565 
566 bool AlkOnlineQuote::launch(const QString &_symbol, const QString &_id, const QString &_source)
567 {
568  if (_source.contains("Finance::Quote")) {
569  return d->launchFinanceQuote(_symbol, _id, _source);
570  } else if (_source.endsWith(".css")) {
571  return d->launchWebKitCssSelector(_symbol, _id, _source);
572  } else if (_source.endsWith(".webkit")) {
573  return d->launchWebKitHtmlParser(_symbol, _id, _source);
574  } else {
575  return d->launchNative(_symbol, _id, _source);
576  }
577 }
578 
580 {
581  return d->m_errors;
582 }
583 
584 #include "alkonlinequote.moc"
bool launchFinanceQuote(const QString &_symbol, const QString &_id, const QString &_source)
AlkOnlineQuotesProfile * profile()
AlkOnlineQuote::Errors m_errors
void slotLoadFinishedHtmlParser(bool ok)
bool launchWebKitCssSelector(const QString &_symbol, const QString &_id, const QString &_source)
bool launchWebKitHtmlParser(const QString &_symbol, const QString &_id, const QString &_source)
Private *const d
bool parseDate(const QString &datestr)
Private(AlkOnlineQuote *parent)
void setProfile(AlkOnlineQuotesProfile *profile)
Errors & operator|=(Type t)
QDate convertString(const QString &date, bool strict=true, unsigned centuryMidPoint=QDate::currentDate().year())
void quote(QString id, QString symbol, QDate date, double price)
bool slotParseQuote(const QString &_quotedata)
void slotLoadFinishedCssSelector(bool ok)
QString getFirstElement(const QString &symbol)
Definition: alkwebpage.cpp:87
static AlkOnlineQuotesProfileManager & instance()
bool initLaunch(const QString &_symbol, const QString &_id, const QString &_source)
bool operator&(Type t) const
bool launchNative(const QString &_symbol, const QString &_id, const QString &_source)
AlkOnlineQuoteProcess m_filter
const Errors & errors()
AlkOnlineQuoteSource m_source
void load(const QUrl &url, const QString &acceptLanguage)
Definition: alkwebpage.cpp:72
AlkOnlineQuote(AlkOnlineQuotesProfile *profile=0, QObject *=0)
bool launch(const QString &_symbol, const QString &_id, const QString &_source=QString())
void setAcceptLanguage(const QString &language)
const QString & what() const
Definition: alkexception.h:73
AlkOnlineQuotesProfile * m_profile
bool parsePrice(const QString &pricestr)