#+PROPERTY: LOGGING nil

* Tasks

** TODO [#A] Document sorters

Note that the built-in sorting only works on Org elements, which is the default ~:action~.  So if a different action is used, sorting will not work.  In that case, the action should be mapped across the Org element results from outside the ~org-ql~ form.

** TODO [#A] Outline path in buffers-files arg
:PROPERTIES:
:ID:       6935361a-9e1d-48ec-8d17-876a90b90f50
:END:

e.g.

#+BEGIN_SRC elisp
  (org-ql (olp "~/org/inbox.org" "Emacs" "Ideas")
    (todo "NEXT"))
#+END_SRC

Also, should support an ~id~ one.

** TODO [#A] Tools for saving queries and accessing them [2/4]

+ Added example to =examples.org=.

*** DONE Save query from ql-agenda buffer

*** DONE Access saved query from saved query list

*** TODO Org link types
:PROPERTIES:
:ID:       4db73c1c-a4ed-425e-9e38-8d334ed03e1e
:END:

This would be useful for having a menu of saved queries as Org links, or even bookmarking saved queries.

**** TODO For all parameters

**** TODO For saved queries

*** TODO Bookmarks

** TODO [#B] Timeline view

e.g. as mentioned by Samuel Wales at https://lists.gnu.org/archive/html/emacs-orgmode/2019-08/msg00330.html.  Prototype code:

#+BEGIN_SRC elisp
  (cl-defun org-ql-timeline (buffers-files query)
    (let ((results
           (org-ql-select buffers-files
             query :action
             (lambda ()
               (let* ((heading-string
                       (->> (org-element-headline-parser
                             (line-end-position))
                            org-ql--add-markers
                            org-ql-agenda--format-element))
                      (timestamps
                       (cl-loop with limit = (org-entry-end-position)
                                while (re-search-forward org-ts-regexp-both
                                                         limit t)
                                collect (ts-parse-org (match-string 0))))
                      (timestamp-strings
                       (->> timestamps
                            (-sort #'ts<)
                            (--map (concat " " (ts-format it))))))
                 (s-join "\n" (cons heading-string timestamp-strings))))
             :sort '(date))))
      (org-ql-agenda--agenda nil nil :strings results)))

  (org-ql-timeline (org-agenda-files)
                   '(and "Emacs" (ts)))

  ;; More timeline-like version, organized by date rather than task.

  (cl-defun org-ql-timeline* (buffers-files query &key filter-ts)
    (let* ((ts-ht (ht))
           (results (org-ql-select buffers-files
                      query
                      :action (lambda ()
                                (let* ((heading-string
                                        (->> (org-element-headline-parser
                                              (line-end-position))
                                             org-ql--add-markers
                                             org-ql-agenda--format-element))
                                       (date-timestamps
                                        ;; Each one set to 00:00:00.
                                        (cl-loop with limit = (org-entry-end-position)
                                                 while (re-search-forward org-ts-regexp-both
                                                                          limit t)
                                                 collect (->> (match-string 0)
                                                              ts-parse-org
                                                              (ts-apply :hour 0 :minute 0 :second 0)))))
                                  (setf date-timestamps (delete-dups date-timestamps))
                                  (when filter-ts
                                    (setf date-timestamps (cl-remove-if-not filter-ts date-timestamps)))
                                  (--each date-timestamps
                                    (push heading-string (gethash it ts-ht)))))))
           (tss-sorted (-sort #'ts< (ht-keys ts-ht)))
           (strings (cl-loop for ts in tss-sorted
                             collect (concat "\n"
                                             (propertize (ts-format "%Y-%m-%d" ts)
                                                         'face 'org-agenda-structure))
                             append (ht-get ts-ht ts))))
      (org-ql-agenda--agenda nil nil :strings strings)))
#+END_SRC

Another, more up-to-date implementation:

#+BEGIN_SRC elisp
  ;; NOTE: ts structs don't (sometimes? or always?) compare properly
  ;; with default hash tables, e.g. this code:

  ;; (let* ((ts-a #s(ts nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil 1572670800.0))
  ;; (ts-b #s(ts nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil nil 1572584400.0)))
  ;; (list :equal (equal ts-a ts-b)
  ;; :sxhash-equal (equal (sxhash ts-a) (sxhash ts-b)))) ;;=> (:equal nil :sxhash-equal t)

  ;; So we must use the "contents-hash" table as described in the Elisp manual.
  (define-hash-table-test 'contents-hash 'equal 'sxhash-equal)

  (cl-defun org-ql-view-timeline (buffers-files &key from to on)
    "FIXME: DOcstring"
    (cl-flet ((parse-ts-arg
               (arg type)
               ;; Parse ARG as a string or TS struct and adjust it to the beginning
               ;; or end of its day, depending on whether TYPE is `:begin' or `:end'.
               (-let (((hour minute second) (cl-ecase type
                                              (:begin '(0 0 0))
                                              (:end '(23 59 59)))))
                 (->> (cl-typecase arg
                        (string (ts-parse arg))
                        (ts arg))
                      (ts-apply :hour hour :minute minute :second second)))))
      (let* ((ts-predicate `(lambda (ts)
                              ,(cond (on `(ts-in ,(parse-ts-arg on :begin)
                                                 ts
                                                 ,(parse-ts-arg on :end)))
                                     ((and from to) `(ts-in ,(parse-ts-arg from :begin)
                                                            ts
                                                            ,(parse-ts-arg to :end)))
                                     (from `(ts<= ,(parse-ts-arg from :begin) ts))
                                     (to `(ts<= ts ,(parse-ts-arg to :end)))
                                     (t (user-error "Huh?")))))
             (query (cond (on `(ts :from ,(parse-ts-arg on :begin)
                                   :to ,(parse-ts-arg on :end)))
                          (t (append (list 'ts)
                                     (when from
                                       `(:from ,(parse-ts-arg from :begin)))
                                     (when to
                                       `(:to ,(parse-ts-arg to :end)))))))
             (date-ts-table (make-hash-table :test 'contents-hash))
             (_results (org-ql-select buffers-files query
                         :action (lambda ()
                                   (let* ((string (->> (org-element-headline-parser
                                                        (line-end-position))
                                                       org-ql--add-markers
                                                       org-ql-view--format-element)))
                                     (cl-loop with limit = (org-entry-end-position)
                                              while (re-search-forward org-ts-regexp-both limit t)
                                              for ts = (->> (match-string 0) ts-parse-org)
                                              when (funcall ts-predicate ts)
                                              do (cl-pushnew (cons ts (concat (ts-format " %H:%M" ts)
                                                                              string))
                                                             (gethash (ts-apply :hour 0 :minute 0 :second 0 ts)
                                                                      date-ts-table)
                                                             :test #'equal))))))
             (date-tss-sorted (->> date-ts-table hash-table-keys (-sort #'ts<)))
             (string (cl-loop for date-ts in date-tss-sorted
                              for date-string = (propertize (ts-format "%Y-%m-%d" date-ts)
                                                            'face 'org-agenda-structure)
                              concat (concat "\n" date-string)
                              concat (cl-loop for (ts . entry) in (->> (gethash date-ts date-ts-table)
                                                                       (-sort (-on #'ts< #'car)))
                                              concat (concat "\n" entry)))))
        (org-ql-view--display :buffer "Timeline"
          :header "Timeline"
          :string string))))

  ;; Used like: 
  ;; (org-ql-view-timeline "~/org/main.org" :from "2019-11-01")
#+END_SRC

** TODO [#B] Add more sorters?

+  [ ] =category=
+  [ ] Any date :: e.g. it would search for timestamps (active/inactive?) anywhere in an entry

** TODO [#B] Default sort

Would probably be useful to have a default sort option.

** TODO [#B] Normalize queries

[2019-07-16 Tue 11:49]  This serves two purposes:

1.  Equivalent queries will return the same results from the cache.
2.  The selectors that can be converted to the fastest preamble regexps will be sorted first, so the fastest preamble will be used.  Although this may not always be straightforward.  For example, in a file with only a few =TODO= items, the ~(todo "TODO")~ selector would convert to a preamble that would quickly search through the file.  But if there were a thousand =TODO= items, it wouldn't be as much of a benefit, and a ~(regexp "something")~ selector's preamble might be much faster, depending on how many times =something= appears in the file.

So the second purpose might actually be a drawback, because it would prevent users from optimizing their queries with knowledge of their data.  Maybe there should be an option to not normalize queries, so advanced users can order their selectors manually.

** TODO [#B] Quickly change sorting/grouping in search views

With caching, the search doesn't need to be repeated, so resorting/regrouping can be very fast.

** TODO [#B] Recursive queries

For lack of a better term.  A way to query for certain headings, and then gather results of a different query at each result of the first query, displaying all results in a single view.  

This works pretty well.  It needs polishing, and some refactoring so items can be indented completely (rather than leaving the keyword unindented, as it is now).

#+BEGIN_SRC elisp
  (cl-defun org-ql-agenda-recursive (buffers-or-files queries &key action narrow sort)
    (cl-labels ((rec (queries element indent)
                     (org-with-point-at (org-element-property :org-marker element)
                       (when-let* ((results (progn
                                              (org-narrow-to-subtree)
                                              (org-ql-select (current-buffer)
                                                (car queries)
                                                :action 'element-with-markers
                                                :narrow t
                                                :sort sort))))
                         ;; Indent entry for each level
                         (setf results (--map
                                        (org-element-put-property it :raw-value
                                                                  (concat (s-repeat (* 5 indent) " ")
                                                                          (org-element-property :raw-value it)))
                                        results))
                         (cons it (if (cdr queries)
                                      (--map (rec (cdr queries) it)
                                             results)
                                    results))))))
      (when-let* ((indent 0)
                  (results (org-ql-select buffers-or-files
                             (car queries)
                             :action 'element-with-markers
                             :narrow narrow
                             :sort sort)))
        (->> (if (cdr queries)
                 (--map (rec (cdr queries) it (1+ indent))
                        results)
               results)
             (-flatten-n (1- (length queries)))
             -non-nil
             (org-ql-agenda--agenda nil nil
               :entries)))))

  (cl-defun org-ql-select-recursive (buffers-or-files queries &key action narrow sort)
    (cl-labels ((rec (queries element indent)
                     (org-with-point-at (org-element-property :org-marker element)
                       (when-let* ((results (progn
                                              (org-narrow-to-subtree)
                                              (org-ql-select (current-buffer)
                                                (car queries)
                                                :action 'element-with-markers
                                                :narrow t
                                                :sort sort))))
                         ;; Indent entry for each level
                         (setf results (--map
                                        (org-element-put-property it :raw-value
                                                                  (concat (s-repeat (* 5 indent) " ")
                                                                          (org-element-property :raw-value it)))
                                        results))
                         (cons it (if (cdr queries)
                                      (--map (rec (cdr queries) it)
                                             results)
                                    results))))))
      (when-let* ((indent 0)
                  (results (org-ql-select buffers-or-files
                             (car queries)
                             :action 'element-with-markers
                             :narrow narrow
                             :sort sort)))
        (->> (if (cdr queries)
                 (--map (rec (cdr queries) it (1+ indent))
                        results)
               results)
             (-flatten-n (1- (length queries)))
             -non-nil))))
#+END_SRC

** TODO [#B] Timeline view
:PROPERTIES:
:ID:       00573552-ffe9-4608-8904-7f6c73246b6d
:END:

e.g. as mentioned by Samuel Wales at https://lists.gnu.org/archive/html/emacs-orgmode/2019-08/msg00330.html.  Prototype code:

#+BEGIN_SRC elisp
  (cl-defun org-ql-timeline (buffers-files query)
    (let ((results
           (org-ql-select buffers-files
             query :action
             (lambda ()
               (let* ((heading-string
                       (->> (org-element-headline-parser
                             (line-end-position))
                            org-ql--add-markers
                            org-ql-agenda--format-element))
                      (timestamps
                       (cl-loop with limit = (org-entry-end-position)
                                while (re-search-forward org-ts-regexp-both
                                                         limit t)
                                collect (ts-parse-org (match-string 0))))
                      (timestamp-strings
                       (->> timestamps
                            (-sort #'ts<)
                            (--map (concat " " (ts-format it))))))
                 (s-join "\n" (cons heading-string timestamp-strings))))
             :sort '(date))))
      (org-ql-agenda--agenda nil nil :strings results)))

  (org-ql-timeline (org-agenda-files)
                   '(and "Emacs" (ts)))

  ;; More timeline-like version, organized by date rather than task.

  (cl-defun org-ql-timeline* (buffers-files query &key filter-ts)
    (let* ((ts-ht (ht))
           (results (org-ql-select buffers-files
                      query
                      :action (lambda ()
                                (let* ((heading-string
                                        (->> (org-element-headline-parser
                                              (line-end-position))
                                             org-ql--add-markers
                                             org-ql-agenda--format-element))
                                       (date-timestamps
                                        ;; Each one set to 00:00:00.
                                        (cl-loop with limit = (org-entry-end-position)
                                                 while (re-search-forward org-ts-regexp-both
                                                                          limit t)
                                                 collect (->> (match-string 0)
                                                              ts-parse-org
                                                              (ts-apply :hour 0 :minute 0 :second 0)))))
                                  (setf date-timestamps (delete-dups date-timestamps))
                                  (when filter-ts
                                    (setf date-timestamps (cl-remove-if-not filter-ts date-timestamps)))
                                  (--each date-timestamps
                                    (push heading-string (gethash it ts-ht)))))))
           (tss-sorted (-sort #'ts< (ht-keys ts-ht)))
           (strings (cl-loop for ts in tss-sorted
                             collect (concat "\n"
                                             (propertize (ts-format "%Y-%m-%d" ts)
                                                         'face 'org-agenda-structure))
                             append (ht-get ts-ht ts))))
      (org-ql-agenda--agenda nil nil :strings strings)))

  (org-ql-timeline* (org-agenda-files)
                    '(ts :from -14)
                    :filter-ts `(lambda (ts)
                                  (ts<= ,(ts-adjust 'day -14 (ts-now)) ts)))
#+END_SRC

[2019-09-26 Thu 21:28]  Would probably make sense to implement this using the view-sections someday.

** TODO [#B] Update view screenshots

e.g. doesn't currently show the =View= header.

** TODO [#C] Test caching

See notes on 1dce9467f25428b5289d3665cd840820969ed65a.  It would be good to test the caching explicitly, at least for some queries, because if I were to completely break it again, in such a way that results were stored but retrieval always failed, the tests wouldn't catch it.

** TODO [#C] ~org-agenda-skip-function~

As discussed [[https://www.reddit.com/r/emacs/comments/cnrt2d/orgqlblock_integrates_orgql_into_org_agenda/ewi1q36/][here]], this is a cool feature that allows further integration into existing custom agenda commands.  Example:

#+BEGIN_SRC elisp
  ;;; lima-0ac22.el --- -*- lexical-binding: t; -*-

  (defun org-ql-skip-function (query)
    "Return a function for `org-agenda-skip-function' for QUERY.
  Compared to using QUERY in `org-ql', this effectively turns QUERY
  into (not QUERY)."
    (let* ((predicate (org-ql--query-predicate '(regexp "ryo-modal"))))
      (lambda ()
        ;; This duplicates the functionality of `org-ql--select'.
        (let (orig-fns)
          (--each org-ql-predicates
            ;; Save original function mappings.
            (let ((name (plist-get it :name)))
              (push (list :name name :fn (symbol-function name)) orig-fns)))
          (unwind-protect
              (progn
                (--each org-ql-predicates
                  ;; Set predicate functions.
                  (fset (plist-get it :name) (plist-get it :fn)))
                ;; Run query.
                ;; FIXME: "If this function returns nil, the current match should not be skipped.
                ;; Otherwise, the function must return a position from where the search
                ;; should be continued."
                (funcall predicate))
            (--each orig-fns
              ;; Restore original function mappings.
              (fset (plist-get it :name) (plist-get it :fn))))))))

  (let ((org-agenda-custom-commands
         '(("z" "Z"
            ((tags-todo "PRIORITY=\"A\"+Emacs/!SOMEDAY"))
            ((org-agenda-skip-function (org-ql-skip-function '(regexp "ryo-modal")))))
           ((org-agenda-files ("~/org/inbox.org"))))))
    (org-agenda nil "z"))
#+END_SRC

I should benchmark it to see how much difference it makes, because all those ~fset~ calls on each heading isn't free.  But if a macro were used to rewrite the built-in predicates to their full versions, all of that could be avoided...

** TODO [#C] Test caching

See notes on 1dce9467f25428b5289d3665cd840820969ed65a.  It would be good to test the caching explicitly, at least for some queries, because if I were to completely break it again, in such a way that results were stored but retrieval always failed, the tests wouldn't catch it.

** TODO [#C] Update commentary

** UNDERWAY [#A] Outline path predicate

[2019-10-07 Mon 11:15]  There are two potential types of matching on outline paths: matching on any part of the outline path, and matching a specific path.  For example, with this file:

#+BEGIN_SRC org
  ,* Food

  ,** Fruits

  ,*** Blueberries

  ,*** Grapes

  ,** Vegetables

  ,*** Carrots

  ,*** Potatoes
#+END_SRC

Matching could work like this:

+  ~(outline "Food")~ :: Would return all nodes.
+  ~(outline "Fruits")~ :: Would return all fruits.

Matching at a specific path would be something like:

+  ~(outline-path "Food" "Fruits")~ :: Would return all fruits.  But if there were another =Fruits= heading somewhere in the file, under a different outline path, it would not return its nodes.

I'm not sure the second type of matching belongs in predicates, but rather in [[id:6935361a-9e1d-48ec-8d17-876a90b90f50][this]].

To implement this with good performance probably needs an outline-path cache.  I can probably repurpose the tags caching, but maybe it should be generalized.

[2019-10-07 Mon 13:09]  This is basically done with =be2bf6df316b96b3ed56851b8ffe0e227796b621= and =be2bf6df316b96b3ed56851b8ffe0e227796b621=, but not the specific-path matching.  I left a =MAYBE= in the code about "anchored" path matching, which would accomplish that.

** UNDERWAY [#B] Implement view with tabulated-list-mode or magit-section

[2019-09-02 Mon 05:20]  Especially with some of the new packages that make =tabulated-list-mode= easier to use, like =navigel=.  However, it would probably break grouping, or require some kind of adapter or extension to do grouping, so I don't know if that would work.  Something like =magit-section= would be more flexible, and could be recursively grouped, like in =magit-todos=.

[2019-09-08 Sun 10:06]  Came up with a prototype yesterday, in branch =wip/view-section=.  Seems to work pretty well.

One of the things in that branch is =org-ql-item=, which is a struct used to carry data for query results.  It seems to work well.

Another idea for it is to simply store the element from =org-element-headline-parser= in one of its slots, and populate all of the other slots lazily, like =ts=.  It already does that for a couple of slots, but I think it makes sense to do it for all of them, to reduce the overhead of making the struct for every query result.

*** TODO Experiment with =widget=

The code that powers the customization UI.  Has collapsible and customizable widgets.  Might be perfect.  Might even enable editing items in the list, with functions to make the changes in the source  buffers.

*** Code idea

Inserting items into a view could look something like this:

#+BEGIN_SRC elisp
  (org-ql-view--insert-items
   :header (ts-format "%Y-%m-%d" (ts-now))
   :items (org-ql-query
            :select #'org-ql-current-item
            :from (org-agenda-files)
            :where '(or (deadline auto)
                        (scheduled :on today)
                        (ts-active :on today)))
   :group-by '(org-ql-item-priority
               org-ql-item-todo))
#+END_SRC

Items would be structs, and the =group-by= argument would be a list of accessors, like how =magit-todos= works.  Arbitrary functions could also be passed to =group-by=, as whatever value the function returns is used to group them.  =org-ql-current-item= would be a function that turns the result of =org-element-headline-parser= into the struct.

Not sure if it should automatically add the number of items to the header, or if that should be done manually.

** UNDERWAY [#B] "Node" caching

[2019-09-05 Thu 12:30]  At each node checked by a predicate, make a struct that stores attributes we can query for, as well as parent node position.  This would let us speed up ancestor-based queries, like =(ancestor (todo "WAITING"))=.  Ideally it would also serve as the tag hierarchy cache.

It would probably be an all-encompassing system, because predicates would need to refer to the cached node when available.  So maybe the struct should be like =ts-defstruct=, with lazy, caching accessors, which would move some of the predicates' code into the accessors.

Maybe a good improvement to make later, after the project is more developed.

[2019-10-07 Mon 13:08]  This has basically been implemented in =be2bf6df316b96b3ed56851b8ffe0e227796b621=, but as functions and values rather than with structs.  It remains to be seen how this works with =ancestor= queries, but I suspect it will help a lot.

*** Struct PoC code

This works okay (except the priority accessor needs to be fixed, because Org priorities are awkward to get).  I'm guessing all the extra function calls would make it undesirable in cases of returning many results, but it's a flexible concept that makes sorting easy.

#+BEGIN_SRC elisp
  (ts-defstruct org-ql-node
    file position marker
    (level
     nil :accessor-init (org-with-point-at (org-ql-node-marker struct)
                          (org-outline-level)))
    (heading
     nil :accessor-init (org-with-point-at (org-ql-node-marker struct)
                          ;; TODO: Org 9.2+ adds 2 more args to `org-get-heading'.
                          (org-get-heading t t)))
    (priority
     nil :accessor-init (org-with-point-at (org-ql-node-marker struct)
                          (org-get-priority )))
    (tags
     nil :accessor-init (org-with-point-at (org-ql-node-marker struct)
                          (->> (org-ql--tags-at (point))
                               -flatten
                               (delq 'org-ql-nil))))
    (todo
     nil :accessor-init (org-with-point-at (org-ql-node-marker struct)
                          (org-get-todo-state)))
    (outline-path
     nil :accessor-init (org-with-point-at (org-ql-node-marker struct)
                          (org-split-string (org-format-outline-path (org-get-outline-path)
                                                                     nil nil "")
                                            ""))))

  (defcustom helm-org-ql-sort
    '(org-ql-node-priority org-ql-node-todo)
    "FIXME"
    )

  (cl-defun helm-org-ql (buffers-files &optional no-and)
    "Display results in BUFFERS-FILES for an `org-ql' query using Helm.
  Interactively, search the current buffer.

  NOTE: Atoms in the query are turned into strings where
  appropriate, which makes it unnecessary to type quotation marks
  around words that are intended to be searched for as indepenent
  strings.

  Also, unless NO-AND is non-nil (interactively, with prefix), all
  query tokens are wrapped in an implied (and) form. This is
  because a query must be a sexp, so when typing multiple clauses,
  either (and) or (or) would be required around them, and (and) is
  typically more useful, because it narrows down results.

  For example, this raw input:

   Emacs git

  Is transformed into this query:

   (and \"Emacs\" \"git\")

  However, quoted strings remain quoted, so this input:

   \"something else\" (tags \"funny\")

  Is transformed into this query:

   (and \"something else\" (tags \"funny\"))"
    (interactive (list (current-buffer) current-prefix-arg))
    (let ((helm-input-idle-delay helm-org-ql-input-idle-delay))
      (helm :sources
            (helm-build-sync-source "helm-org-ql-agenda-files"
              :candidates (lambda ()
                            (let* ((query (helm-org-ql--input-to-query helm-pattern no-and))
                                   (window-width (window-width (helm-window))))
                              (when query
                                (let ((results (org-ql-select buffers-files
                                                 query
                                                 :action '(make-org-ql-node :marker (point-marker)))))
                                  (when helm-org-ql-sort
                                    (dolist (sorter (reverse helm-org-ql-sort))
                                      (setf results (sort results sorter))))
                                  (cl-loop for it in-ref results
                                           do (setf it (concat (buffer-name (org-ql-node-file it)) ":"
                                                               (or (org-ql-node-todo it) "")
                                                               (or (org-ql-node-priority it) "")
                                                               (org-ql-node-heading it) "\\"
                                                               (org-ql-node-outline-path it))))
                                  results))))
              :match #'identity
              :fuzzy-match nil
              :multimatch nil
              :volatile t
              :action #'helm-org-goto-marker))))
#+END_SRC

** MAYBE Overlay-based caching inspired by org-num-mode

[2019-12-30 Mon 22:42]  Newer versions of Org have =org-num-mode=, which uses =font-lock= and =after-change-functions= to update overlays in the buffer with outline numbering.  Maybe a similar approach could be used to cache arbitrary values for headings in a buffer without having to discard the whole buffer's cache when the buffer changes.

** MAYBE [#C] Fancier searching for inherited tags

When tag inheritance is enabled, and the given tags aren't file-level tags, we could search directly to headings containing the matching tags, and then only do per-heading matching on the subtrees.  Sometimes that would be much faster.  However, that might make the logic special-cased and complicated.  Might need a redesign of the whole matching/predicate system to do cleanly.

** DONE [#A] Helm command

In branch =wip/helm-org-ql=.  Works really well, should add it and demonstrate it.

*** DONE Add

*** DONE Demonstrate

*** DONE Parsing non-Lisp queries

[2019-09-12 Thu 12:56]  Lisp is so much easier to deal with, but some people don't like parentheses.  So I'm trying to add a non-Lisp-style query syntax.  It gets complicated.  The =peg= library helps, but its documentation is sparse and incomplete.  This seems to work fairly well for single-token queries, but I'm not sure if I can or should cram it all into one parser, or use separate ones for certain keywords.

#+BEGIN_SRC elisp
  (-let* ((input "todo:check|someday")
          (input "tags:universe+space")
          (input "heading:\"spaced phrase\"")
          (input "")
          (input "heading:\"spaced phrase\"+another")
          combinator
          (parsed (peg-parse-string ((predicate (substring keyword) ":" (opt args))
                                     (keyword (or "heading" "tags" "todo" "property"))
                                     (args (+ (and (or quoted-arg unquoted-arg) (opt separator))))
                                     (quoted-arg "\"" unquoted-arg "\"")
                                     (unquoted-arg (substring (+ (not (or separator "\"")) (any))))
                                     (separator (or (and "|" (action (setf combinator 'or)))
                                                    (and "+" (action (setf combinator 'and)))
                                                    (and ":" (action (setf combinator 'arg))))))
                                    input 'noerror))
          ((predicate . args) (nreverse parsed)))
    (when predicate
      (list :predicate predicate :args args :combinator combinator)))
  ;;=> (:predicate "heading" :args ("spaced phrase" "another" t) :combinator and)
#+END_SRC

I don't know where the =t= is coming from.

The next step is to make it work with multi-token queries.  It needs to handle all of the tokens in one parser so it can handle quoted phrases (if we split on spaces, it would split quoted phrases).  But that makes getting the arguments out of it more difficult.  Probably need to do something like this:

#+BEGIN_SRC elisp
  (-let* ((input "todo:check|someday")
          (input "tags:universe+space")
          (input "heading:\"spaced phrase\"")
          (input "")
          (input "heading:\"spaced phrase\"+another")
          combinator
          (parsed (peg-parse-string ((query (+ (or (and predicate `(pred args -- (list :predicate pred :args args)))
                                                   (and plain-string `(s -- (list :predicate 'regexp :args s))))
                                               (opt (syntax-class whitespace))))
                                     (plain-string (substring (+ (not (syntax-class whitespace)) (any))))
                                     (predicate (substring keyword) ":" (opt args))
                                     (keyword (or "heading" "tags" "todo" "property"))
                                     (args (+ (and (or quoted-arg unquoted-arg) (opt separator))))
                                     (quoted-arg "\"" unquoted-arg "\"")
                                     (unquoted-arg (substring (+ (not (or separator "\"")) (any))))
                                     (separator (or (and "|" (action (setf combinator 'or)))
                                                    (and "+" (action (setf combinator 'and)))
                                                    (and ":" (action (setf combinator 'arg))))))
                                    input 'noerror)))
    parsed)
#+END_SRC

In which lists are pushed onto the stack and returned, rather than strings.  But I don't understand yet exactly how to use the =var= forms to consume input from the "value stack"; I need to study the examples more.  I'm also not sure if that will even work with a variable number of arguments.

This seems to work, but we'll have to parse the args again in a separate step:

#+BEGIN_SRC elisp
  (-let* ((input "todo:check|someday")
          (input "tags:universe+space")
          (input "heading:\"spaced phrase\"")
          (input "")
          (input "heading:\"spaced phrase\"+another")
          (input "heading:\"spaced phrase\"+another todo:check")
          combinator
          (parsed (peg-parse-string ((query (+ (or (and predicate `(pred args -- (list :predicate pred :args args)))
                                                   (and plain-string `(s -- (list :predicate 'regexp :args s))))
                                               (opt (+ (syntax-class whitespace) (any)))))
                                     (plain-string (substring (+ (not (syntax-class whitespace)) (any))))
                                     (predicate (substring keyword) ":" (opt args))
                                     (keyword (or "heading" "tags" "todo" "property"))
                                     (args (substring (+ (and (or quoted-arg unquoted-arg) (opt separator)))))
                                     (quoted-arg "\"" (+ (not (or separator "\"")) (any)) "\"")
                                     (unquoted-arg (+ (not (or separator "\"" (syntax-class whitespace))) (any)))
                                     (separator (or (and "|" (action (setf combinator 'or)))
                                                    (and "+" (action (setf combinator 'and)))
                                                    (and ":" (action (setf combinator 'arg))))))
                                    input 'noerror)))
    parsed)
    ;;=> (t (:predicate "todo" :args "check") (:predicate "heading" :args "\"spaced phrase\"+another"))
#+END_SRC

Well, a bit of fiddling (lots of trial-and-error required) produced this:

#+BEGIN_SRC elisp
  (-let* ((input "todo:check|someday")
          (input "tags:universe+space")
          (input "heading:\"spaced phrase\"")
          (input "")
          (input "heading:\"spaced phrase\"+another")
          (input "heading:\"spaced phrase\"+another todo:check")
          combinator
          (parsed (peg-parse-string ((query (+ (or (and predicate `(pred args -- (list :predicate pred :args args)))
                                                   (and plain-string `(s -- (list :predicate 'regexp :args s))))
                                               (opt (+ (syntax-class whitespace) (any)))))
                                     (plain-string (substring (+ (not (syntax-class whitespace)) (any))))
                                     (predicate (substring keyword) ":" (opt args))
                                     (keyword (or "heading" "tags" "todo" "property"))
                                     (args (list (+ (and (substring (or quoted-arg unquoted-arg)) (opt separator)))))
                                     (quoted-arg "\"" (+ (not (or separator "\"")) (any)) "\"")
                                     (unquoted-arg (+ (not (or separator "\"" (syntax-class whitespace))) (any)))
                                     (separator (or (and "|" (action (setf combinator 'or)))
                                                    (and "+" (action (setf combinator 'and)))
                                                    (and ":" (action (setf combinator 'arg))))))
                                    input 'noerror)))
    parsed)
    ;;=> (t (:predicate "todo" :args ("check")) (:predicate "heading" :args ("\"spaced phrase\"" "another")))
#+END_SRC

That seems pretty usable!

** DONE Byte-compile lambdas
CLOSED: [2018-05-09 Wed 17:30]
:LOGBOOK:
-  State "DONE"       from              [2018-05-09 Wed 17:30]
:END:

=elfeed-search--update-list= byte-compiles lambdas returned by =elfeed-search-compile-filter=.  Maybe I could do something like this too.

If I can get this working, I should profile it to see what difference it makes.

*** Profiling

Going to try byte-compiling the predicate function:

#+BEGIN_SRC elisp
  (elp-profile 10 nil (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
                   (and (or (date :date '= (org-today))
                            (date :deadline '<= (+ org-deadline-warning-days (org-today)))
                            (date :scheduled '<= (org-today)))
                        (not (apply #'todo org-done-keywords-for-agenda)))))
#+END_SRC

#+RESULTS:
| Function                                  | Times called |   Total time | Average time |
|-------------------------------------------+--------------+--------------+--------------|
| org-agenda-ng--agenda                     |           10 | 0.8370581039 | 0.0837058104 |
| org-agenda-finalize-entries               |           10 |  0.652886608 | 0.0652886608 |
| org-super-agenda--filter-finalize-entries |           10 |  0.641794501 | 0.0641794501 |
| org-super-agenda--group-items             |           10 |  0.636057006 | 0.0636057006 |
| org-super-agenda--group-dispatch          |          130 |  0.631911849 | 0.0048608603 |
| org-super-agenda--group-tag               |           50 |  0.592883869 | 0.0118576773 |
| list                                      |         2720 | 0.5792795169 | 0.0002129704 |
| mapcar                                    |          331 | 0.2333591920 | 0.0007050126 |
| org-agenda-ng--filter-buffer              |           10 |   0.09247626 |  0.009247626 |
| org-agenda-ng--format-element             |          150 | 0.0649320479 | 0.0004328803 |
| org-entry-get                             |          860 | 0.0408285349 | 4.747...e-05 |
| org-agenda-ng--date-p                     |          910 | 0.0385646249 | 4.237...e-05 |
| org-element-headline-parser               |          150 | 0.0374417470 | 0.0002496116 |
| org-is-habit-p                            |          270 | 0.0290107389 | 0.0001074471 |
| org--property-local-values                |          270 | 0.0268615979 | 9.948...e-05 |
| org-get-property-block                    |          270 | 0.0244613309 | 9.059...e-05 |
| org-get-tags-at                           |          150 |  0.017875864 | 0.0001191724 |
| org-super-agenda--group-habit             |           10 |  0.015910656 | 0.0015910655 |
| mapc                                      |         2540 | 0.0158616290 | 6.244...e-06 |
| org-agenda-ng--add-faces                  |          150 | 0.0143329670 | 9.555...e-05 |


Now the same thing without byte-compiling:

#+BEGIN_SRC elisp
  (elp-profile 10 nil (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
                   (and (or (date :date '= (org-today))
                            (date :deadline '<= (+ org-deadline-warning-days (org-today)))
                            (date :scheduled '<= (org-today)))
                        (not (apply #'todo org-done-keywords-for-agenda)))))
#+END_SRC

#+RESULTS:
| Function                                  | Times called |   Total time | Average time |
|-------------------------------------------+--------------+--------------+--------------|
| org-agenda-ng--agenda                     |           10 |  0.846645537 | 0.0846645537 |
| org-agenda-finalize-entries               |           10 |  0.662896805 | 0.0662896805 |
| sort                                      |           40 |  0.591123256 | 0.0147780814 |
| org-entries-lessp                         |          400 | 0.5901201620 | 0.0014753004 |
| mapcar                                    |          201 | 0.2318270599 | 0.0011533684 |
| org-agenda-ng--filter-buffer              |           10 |  0.092519787 | 0.0092519787 |
| org-super-agenda--filter-finalize-entries |           10 | 0.0664278040 | 0.0066427804 |
| org-agenda-ng--format-element             |          150 |  0.064658994 | 0.0004310599 |
| org-super-agenda--group-items             |           10 | 0.0602504089 | 0.0060250408 |
| org-super-agenda--group-dispatch          |          130 | 0.0561904470 | 0.0004322342 |
| org-entry-get                             |          860 | 0.0437458889 | 5.086...e-05 |
| org-agenda-ng--date-p                     |          910 | 0.0382623409 | 4.204...e-05 |
| org-element-headline-parser               |          150 | 0.0374662920 | 0.0002497752 |
| org-is-habit-p                            |          270 | 0.0320861079 | 0.0001188374 |
| org--property-local-values                |          270 | 0.0298690430 | 0.0001106260 |
| org-get-property-block                    |          270 | 0.0274716649 | 0.0001017469 |
| org-super-agenda--group-habit             |           10 |  0.019117901 | 0.0019117901 |
| org-get-tags-at                           |          150 | 0.0178958930 | 0.0001193059 |
| mapc                                      |         2470 | 0.0150361130 | 6.087...e-06 |
| org-agenda-ng--add-faces                  |          150 | 0.0143092169 | 9.539...e-05 |

Virtually indistinguishable.  Going to try moving the =byte-compile= call from the =org-agenda-ng= macro to other places...

#+BEGIN_SRC elisp
  (elp-profile 10 nil (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
                   (and (or (date :date '= (org-today))
                            (date :deadline '<= (+ org-deadline-warning-days (org-today)))
                            (date :scheduled '<= (org-today)))
                        (not (apply #'todo org-done-keywords-for-agenda)))))
#+END_SRC

#+RESULTS:
| Function                                  | Times called |   Total time | Average time |
|-------------------------------------------+--------------+--------------+--------------|
| org-agenda-ng--agenda                     |           10 | 0.8476316779 | 0.0847631678 |
| mapcar                                    |          331 | 0.8159452220 | 0.0024650913 |
| org-agenda-ng--filter-buffer              |           10 |  0.674217912 | 0.0674217912 |
| org-element-headline-parser               |          150 | 0.6171195889 | 0.0041141305 |
| line-beginning-position                   |          620 | 0.5802579680 | 0.0009358999 |
| org-agenda-finalize-entries               |           10 |  0.082065157 | 0.0082065157 |
| org-super-agenda--filter-finalize-entries |           10 | 0.0708772279 | 0.0070877227 |
| org-super-agenda--group-items             |           10 |  0.065523103 | 0.0065523103 |
| org-agenda-ng--format-element             |          150 | 0.0652783740 | 0.0004351891 |
| org-super-agenda--group-dispatch          |          130 | 0.0614253589 | 0.0004725027 |
| org-entry-get                             |          860 | 0.0494023029 | 5.744...e-05 |
| org-agenda-ng--date-p                     |          910 | 0.0388435519 | 4.268...e-05 |
| org-is-habit-p                            |          270 | 0.0375687549 | 0.0001391435 |
| org--property-local-values                |          270 | 0.0353892929 | 0.0001310714 |
| org-get-property-block                    |          270 | 0.0329700440 | 0.0001221112 |
| org-super-agenda--group-habit             |           10 |  0.024468601 | 0.0024468601 |
| re-search-backward                        |         1500 | 0.0186344089 | 1.242...e-05 |
| org-get-tags-at                           |          150 | 0.0180038809 | 0.0001200258 |
| mapc                                      |         2540 | 0.0156518099 | 6.162...e-06 |
| org-agenda-ng--add-faces                  |          150 | 0.0144141080 | 9.609...e-05 |

Doesn't seem to make any difference.

** DONE Document/figure out tag inheritance

I think it should probably be enabled in most cases, to avoid missing results that users would expect to find, but it will reduce performance in some cases, so users should be able to turn it off when they don't need it.

[2018-06-12 Tue 14:32]  The docstring for ~org-map-entries~ says:

#+BEGIN_QUOTE
If your function needs to retrieve the tags including inherited tags at the *current* entry, you can use the value of the variable ‘org-scanner-tags’ which will be much faster than getting the value with ‘org-get-tags-at’.  If your function gets properties with ‘org-entry-properties’ at the *current* entry, bind ‘org-trust-scanner-tags’ to t around the call to ‘org-entry-properties’ to get the same speedup.  Note that if your function moves around to retrieve tags and properties at a *different* entry, you cannot use these techniques.
#+END_QUOTE

[2019-09-26 Thu 21:31]  Handled with the tag caching recently implemented.

** DONE [#B] Dual matching with regexp and predicates
:PROPERTIES:
:ID:       39972bb5-fdd0-4754-93ba-c85796a67ccf
:END:

/Note: This is underway in the =preamble-re= branch./

Searching and matching could be sped up by constructing a regexp that searches directly to the next possible match, and then matching with predicate functions.

For example, a search like:

#+BEGIN_SRC elisp
  (org-ql (org-agenda-files)
    (and (regexp "lisp")
         (scheduled < today)))
#+END_SRC

Only entries that contain the word =lisp= can be matches, and searching each entry for that word is wasteful.  Instead, we could search the buffer for the next occurrence of =lisp=, then check the scheduled date for that entry.

This would require processing the predicate to pull out matchers that can be done as buffer-wide regexps, e.g. =regexp=, =heading-regexp=, =todo=, and possibly =tags=.  Org has some regexp-building functions that might make this fairly easy, and then we could probably use ~rx~ to make an optimized version of the regexp.  It would also require some refactoring to the searching that would go directly to regexp matches when possible, rather than checking every entry with the predicate.

[2019-07-16 Tue 11:14]  Made new branch =preamble-re-new= based on current =master=.  Seems to work well.  Here's some code for testing and comparing performance (~bench-multi-lets~ is from [[https://github.com/alphapapa/emacs-package-dev-handbook#bench-multi-lets][here]]).

[2019-07-16 Tue 11:56]  Going to merge to =master= as 0.2, so marking this as done, even though there's a bit more that can be done from here.

*** Benchmark code

#+BEGIN_SRC elisp
  (cl-defmacro org-ql-preamble-bench (&key query (file "tests/data.org") (times 10))
    `(bench-multi-lets :times ,times :ensure-equal t
       :lets (("preamble" ((org-ql-use-preamble t)))
              ("no preamble" ((org-ql-use-preamble nil))))
       :forms ((,(prin1-to-string query) (org-ql-select,file
                                          ',query
                                          :action (lambda () (org-get-heading t t)))))))
#+END_SRC

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :query (regexp "Emacs") :times 100)
#+END_SRC

#+RESULTS:
| Form                          | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-------------------------------+--------------------+---------------+----------+------------------|
| preamble: (regexp "Emacs")    | 1.22               |      0.141767 |        0 |                0 |
| no preamble: (regexp "Emacs") | slowest            |      0.172398 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :file "~/org/inbox.org" :query (regexp "Emacs") :times 5)
#+END_SRC

#+RESULTS:
| Form                          | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-------------------------------+--------------------+---------------+----------+------------------|
| preamble: (regexp "Emacs")    | 1.59               |      2.011043 |        0 |                0 |
| no preamble: (regexp "Emacs") | slowest            |      3.206370 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :file "~/org/inbox.org" :query (and (regexp "Emacs") (todo)) :times 5)
#+END_SRC

#+RESULTS:
| Form                                       | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (and (regexp "Emacs") (todo))    | 1.59               |      2.211503 |        0 |                0 |
| no preamble: (and (regexp "Emacs") (todo)) | slowest            |      3.512741 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :file "~/org/inbox.org" :query (and (regexp "Emacs") (todo) (scheduled)) :times 5)
#+END_SRC

#+RESULTS:
| Form                                                   | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (and (regexp "Emacs") (todo) (scheduled))    | 1.69               |      2.042456 |        0 |                0 |
| no preamble: (and (regexp "Emacs") (todo) (scheduled)) | slowest            |      3.453756 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :file "~/org/inbox.org" :query (todo "WAITING") :times 2)
#+END_SRC

#+RESULTS:
| Form                          | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-------------------------------+--------------------+---------------+----------+------------------|
| preamble: (todo "WAITING")    | 15.60              |      0.070684 |        0 |                0 |
| no preamble: (todo "WAITING") | slowest            |      1.102722 |        0 |                0 |

Wow, that's a huge improvement!

** DONE Operate on list of heading positions
CLOSED: [2018-05-10 Thu 15:02]
:LOGBOOK:
-  State "DONE"       from              [2018-05-10 Thu 15:02]
:END:

[2017-12-31 Sun 17:54]  I wonder if, instead of parsing the whole buffer with =org-element-parse-buffer=, we could simply work on a list of heading positions, e.g. a loop would search forward to the next heading position, then call whatever predicates it needed at the heading's position, using =save-excursion= around each function call.  The predicates would need to be updated to get their data from the buffer, instead of using =org-element-property=, but that wouldn't be hard.

[2018-05-10 Thu 15:01]  I already changed to using buffer-parsing predicates instead of =org-element-parse-buffer=.

** DONE Use macros for =date=
CLOSED: [2018-05-10 Thu 14:59]
:LOGBOOK:
-  State "DONE"       from              [2018-05-10 Thu 14:59]
:END:

If I made the =date= selector a macro, I could avoid the need to quote the comparator.

Also, maybe instead of having a single =date= selector, I should have =scheduled=, =deadline=, etc.

* References

** [[https://m00natic.github.io/lisp/manual-jit.html][Uniform Structured Syntax, Metaprogramming and Run-time Compilation]]

** [[gnus:gmane.emacs.orgmode#CAJ51EToLCm5zDLKu8XeuqEWrLhHZF+OoNkviPSivZbFttzF8=A@mail.gmail.com][John Kitchin on rewriting the Org agenda code]]                 :Emacs:Org:

[2019-10-28 Mon 08:28]  Originally from [[id:90a68535-2403-4ba9-a117-60be1862628f][this entry in my notes]].

#+BEGIN_QUOTE
From: John Kitchin <jkitchin@andrew.cmu.edu>
Subject: Re: [Orgmode] Slow speed of week and month views
Newsgroups: gmane.emacs.orgmode
To: Karl Voit <news1142@karl-voit.at>
Cc: "emacs-orgmode@gnu.org" <emacs-orgmode@gnu.org>
Date: Sat, 5 Aug 2017 18:17:09 -0400 (4 hours, 11 minutes, 38 seconds ago)

I can think of two possibilities for a future approach (besides a deep dive on profiling the current elisp to improve the speed there). They both involve some substantial coding though, and would probably add dependencies. I am curious what anyone things about these, or if there are other ideas.

One is to use the new dynamic module capability to write an org parser in C, or a dedicated agenda function, which would presumably be faster than in elisp. This seems hard, and for me would certainly be a multiyear project I am sure! The downside of this is the need to compile the module. I don't know how easy it would be to make this work across platforms with the relatively easy install org-mode currently has. This could have a side benefit though of a c-lib that could be used by others to expand where org-mode is used.

The other way that might work is to rely more heavily on a cached version of the files, perhaps in a different format than elisp, that is faster to work with.  The approach I have explored in this is to index org files into a sqlite database. The idea then would be to generate the agenda from a sql query. I use something like this already to "find stuff in orgmode anywhere". One of the reasons I wrote this is the org-agenda list of files isn't practical for me because my files are so scattered on my file system. I had a need to be able to find TODOs in research projects in a pretty wide range of locations.

The code I use is at https://github.com/jkitchin/scimax/blob/master/org-db.el, and from one database I can find headlines, contacts, locations, TODO headlines across my file system, all the files that contain a particular link, and my own recent org files. This approach relies on emacsql, and a set of hook functions to update the database whenever a file is changed. It is not robust, e.g. the file could be out of sync with the db if it is modified outside emacs, but this works well enough for me so far. Updated files get reindexed whenever emacs is idle. It was a compromise on walking the file system all the time or daily, or trying to use inotify and you can always run a command to prune/sync all the files any time you want.

sqlite is ok, but with emacsql you cannot put strings in it directly (at least when I wrote the org-db code), which has limited it for full-text search so far.  Also with text, the db got up to about 0.5 GB in size, and started slowing down. So it doesn't have text in it for now. It has all the other limitations of sqlite too, limited support for locking, single process....

I am moderately motivated to switch from sqlite to MongoDB, but the support for Mongo in emacs is pretty crummy (I tried writing a few traditional interfaces, but the performance was not that good, and limited since Mongo uses bson, and it is just not the same as json!). Why Mongo? Mostly because the Mongo query language is basically json and easy to generate in Emacs, unlike sql. Also, it is flexible and easy to adapt to new things, e.g. indexing src-blocks or tables or whatever org-element you want. (And I want to use Mongo for something else too ;). Obviously these all add dependencies, and might not be suitable for the core org-mode distribution. But I do think it is important to think about ways to scale org-mode while maintaining compatibility with the core.

The main point of the database was to get a query language, persistence and good performance. I have also used caches to speed up using bibtex files, and my org-contacts with reasonable performance. These have been all elisp, with no additional dependencies. Maybe one could do something similar to keep an agenda cache that is persistent and updated via hook functions.

Thoughts?

John
#+END_QUOTE
:PROPERTIES:
:archive.is: http://archive.is/33R9M
:END:

+ [[https://github.com/m00natic/cl-fdbq][GitHub - m00natic/cl-fdbq: SQL-like operations over fixed field DBs]]

** Nicolas Goaziou's =org-element= cache implementation

[2020-01-04 Sat 08:31]  https://code.orgmode.org/bzg/org-mode/src/master/lisp/org-element.el#L4817  I guess I haven't been keeping up, because I'm not sure when exactly he made or committed this, or what his plans for it are.  It's currently disabled by default.

** [[https://github.com/ndwarshuis/org-sql][GitHub - ndwarshuis/org-sql: SQL backend for Emacs Org-Mode]]

[2020-01-04 Sat 09:03]

* Examples / testing

#+BEGIN_SRC elisp
  (org-agenda-ng org-agenda-files
    (and (or (date :deadline '<= (org-today))
             (date :scheduled '<= (org-today)))
         (not (apply #'todo org-done-keywords-for-agenda)))
    ((group (tags "bills"))
     (group (todo "SOMEDAY"))))

  (org-agenda-ng org-agenda-files
    (and (or (date :deadline '<= (org-today))
             (date :scheduled '<= (org-today)))
         (not (apply #'todo org-done-keywords-for-agenda))))

  (org-agenda-ng "~/org/main.org"
    (and (or (date :deadline '<= (org-today))
             (date :scheduled '<= (org-today)))
         (not (apply #'todo org-done-keywords-for-agenda))))

  (org-ql org-agenda-files
    (and (todo "SOMEDAY")
         (tags "Emacs")))
  (org-ql org-agenda-files
    (and (todo "SOMEDAY")
         (tags "Emacs")
         (priority >= "B")))
  (org-ql "~/org/main.org"
    (and (or (tags "Emacs")
             (priority >= "B"))
         (not (done))))
  (org-ql "~/org/main.org"
    (and (or (tags "Emacs")
             (priority >= "B"))
         (done)))
#+END_SRC

** Sorting

#+BEGIN_SRC elisp
  (org-ql "~/src/emacs/org-super-agenda/test/test.org"
    (regexp "over")
    :sort (priority deadline scheduled))

  (org-ql "~/src/emacs/org-super-agenda/test/test.org"
    (regexp "over")
    :sort (date))

  (org-ql "~/src/emacs/org-super-agenda/test/test.org"
    (todo)
    :sort (todo))
#+END_SRC

** Regexp matching

#+BEGIN_SRC elisp
  (org-ql "~/src/emacs/org-super-agenda/test/test.org"
    (regexp "over"))

  (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
    (regexp "over"))
#+END_SRC

** Property matching

#+BEGIN_SRC elisp
  (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
    (property "agenda-group"))

  (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
    (property "agenda-group" "plans"))
#+END_SRC

** Screenshot code

#+BEGIN_SRC elisp
  (org-super-agenda--test-with-org-today-date "2017-07-08 00:00"
    (org-ql "~/src/emacs/org-super-agenda/test/test.org"
      (and (or (date = today)
               (deadline <=)
               (scheduled <= today))
           (not (done)))))
#+END_SRC

* In the wild

** [[https://github.com/AloisJanicek/.doom.d-2nd][Alois Janicek]]

*** [[https://github.com/AloisJanicek/.doom.d-2nd/commit/41ed1080f6f90463fc1f1d7e47cef9864756867c][further tweaking org-ql views · AloisJanicek/.doom.d-2nd@41ed108 · GitHub]]

#+BEGIN_SRC elisp :eval never
;; Hydras
(defhydra gtd-agenda (:color blue
                             :body-pre
                             (if (aj/has-heading-p +INBOX)
                                 (org-ql-search `(,+INBOX) "*"
                                   :sort '(date))
                               (org-ql-search (org-agenda-files)
                                 '(todo "NEXT")
                                 :sort '(date priority todo)
                                 :groups '((:auto-category t))))
                             )
  "agenda"
  ("a" (org-ql-agenda (org-agenda-files)
         (and (or (ts-active :on today)
                  (deadline auto)
                  (scheduled :to today))
              (not (done)))) "agenda")

  ("t" (org-ql-search (org-agenda-files)
         '(todo "TODO")
         :sort '(date priority todo)
         :groups '((:auto-category t))) "todo")

  ("n" (org-ql-search (org-agenda-files)
         '(todo "NEXT")
         :sort '(date priority todo)
         :groups '((:auto-category t))) "next")

  ("h" (org-ql-search (org-agenda-files)
         '(todo "HOLD")
         :sort '(date priority todo)
         :groups '((:auto-category t))) "hold")

  ("s" (org-ql-search (org-agenda-files)
         '(tags "someday")
         :sort '(date priority todo)
         :groups '((:auto-category t))) "someday")

  ("r" (org-ql-search (org-agenda-files)
         '(ts :from -7 :to today)
         :sort '(date priority todo)
         :groups '((:auto-ts t))) "recent")

  ("i" (org-ql-search `(,+INBOX) "*"
         :sort '(date)) "inbox")
  )
#+END_SRC

** [[https://github.com/pestctrl/emacs-config][Benson Chu]]

*** [[https://github.com/pestctrl/emacs-config/commit/fa3068003373a0c93e23c728b5dbad2d7c11e2e1][Experimental agenda view with org-ql · pestctrl/emacs-config@fa30680 · GitHub]]

#+BEGIN_SRC elisp :eval never
              ("f" "fastdev?"
               ((org-ql-block '(tags "refile")
                              ((org-agenda-overriding-header "Refile tasks")))
                (tags-todo ,(concat dev-tag "/!" (mapconcat #'identity my/active-projects-and-tasks "|"))
                           ((org-agenda-overriding-header "Stuck Projects")
                            (org-agenda-skip-function 'my/dev-show-stuck-projects)
                            (org-tags-match-list-sublevels 'indented)
                            (org-agenda-sorting-strategy
                             '((agenda category-keep)))))
                (tags-todo ,(concat dev-tag "-short" "/!" (mapconcat #'identity my/active-projects-and-tasks "|"))
                           ((org-agenda-overriding-header "Active Projects")
                            (org-agenda-skip-function 'my/dev-show-active-projects)
                            (org-tags-match-list-sublevels 'indented)
                            (org-agenda-sorting-strategy
                             '((agenda category-keep)))))
                (org-ql-block '(and (tags "dev")
                                    (todo "WAIT"))
                              ((org-agenda-overriding-header "Waiting tasks")))
                (org-ql-block '(and (tags "dev")
                                    (todo "NEXT"))
                              ((org-agenda-overriding-header "Things to do")))
                (agenda ""
                        ((org-agenda-skip-function 'my/agenda-custom-skip)
                         (org-agenda-span 'day)
                         (org-agenda-tag-filter-preset (quote (,dev-tag)))
                         (org-agenda-skip-deadline-if-done t)
                         (org-agenda-skip-scheduled-if-done t)
                         (org-super-agenda-groups '((:name "Overdue" :and (:deadline past :log nil))
                                                    (:name "Upcoming" :deadline future)
                                                    (:name "Should do" :and (:scheduled past :log nil))
                                                    (:name "Today" :time-grid t
                                                           :and (:not (:and (:not (:scheduled today)
                                                                                  :not (:deadline today)))))))))))
#+END_SRC

* Profiling

** Preambles

Not sure if clearing the cache is necessary here, because it seemed to make nearly no difference in the results, but I don't know why.

#+BEGIN_SRC elisp :results silent
  (cl-defmacro org-ql-preamble-bench (&key query (file "tests/data.org") (times 10))
    `(bench-multi-lets :times ,times :ensure-equal t
       :lets (("preamble" ((org-ql-use-preamble t)
                           (org-ql-cache (ht))))
              ("no preamble" ((org-ql-use-preamble nil)
                              (org-ql-cache (ht)))))
       :forms ((,(prin1-to-string query) (org-ql-select ,file
                                           ',query
                                           :action '(org-get-heading t t))))))
#+END_SRC

*** =closed=

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (closed))
#+END_SRC

#+RESULTS:
| Form                  | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-----------------------+--------------------+---------------+----------+------------------|
| preamble: (closed)    | 4.80               |      0.086553 |        0 |                0 |
| no preamble: (closed) | slowest            |      0.415165 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (closed <= "2019-01-01"))
#+END_SRC

#+RESULTS:
| Form                                  | x faster than next | Total runtime | # of GCs | Total GC runtime |
|---------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (closed <= "2019-01-01")    | 4.21               |      0.105782 |        0 |                0 |
| no preamble: (closed <= "2019-01-01") | slowest            |      0.445374 |        0 |                0 |

*** =deadline=

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (deadline))
#+END_SRC

#+RESULTS:
| Form                    | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-------------------------+--------------------+---------------+----------+------------------|
| preamble: (deadline)    | 27.63              |      0.014656 |        0 |                0 |
| no preamble: (deadline) | slowest            |      0.404952 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (deadline <= "2019-01-01"))
#+END_SRC

#+RESULTS:
| Form                                    | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-----------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (deadline <= "2019-01-01")    | 27.91              |      0.014606 |        0 |                0 |
| no preamble: (deadline <= "2019-01-01") | slowest            |      0.407682 |        0 |                0 |

*** =habit=

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (habit))
#+END_SRC

#+RESULTS:
| Form                 | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------------------+--------------------+---------------+----------+------------------|
| preamble: (habit)    | 70.09              |      0.016489 |        0 |                0 |
| no preamble: (habit) | slowest            |      1.155649 |        0 |                0 |

*** =level=

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (level 1))
#+END_SRC

#+RESULTS:
| Form                   | x faster than next | Total runtime | # of GCs | Total GC runtime |
|------------------------+--------------------+---------------+----------+------------------|
| preamble: (level 1)    | 1.34               |      0.562950 |        0 |                0 |
| no preamble: (level 1) | slowest            |      0.754050 |        0 |                0 |

*** =property=

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (property "agenda-group"))
#+END_SRC

#+RESULTS:
| Form                                   | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (property "agenda-group")    | 70.44              |      0.016571 |        0 |                0 |
| no preamble: (property "agenda-group") | slowest            |      1.167203 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (property "ID"))
#+END_SRC

#+RESULTS:
| Form                         | x faster than next | Total runtime | # of GCs | Total GC runtime |
|------------------------------+--------------------+---------------+----------+------------------|
| preamble: (property "ID")    | 3.51               |      0.369830 |        0 |                0 |
| no preamble: (property "ID") | slowest            |      1.299684 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (property "agenda-group" "plans"))
#+END_SRC

#+RESULTS:
| Form                                           | x faster than next | Total runtime | # of GCs | Total GC runtime |
|------------------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (property "agenda-group" "plans")    | 72.54              |      0.016862 |        0 |                0 |
| no preamble: (property "agenda-group" "plans") | slowest            |      1.223197 |        0 |                0 |

*** =scheduled=

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (scheduled))
#+END_SRC

#+RESULTS:
| Form                     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------+--------------------+---------------+----------+------------------|
| preamble: (scheduled)    | 4.45               |      0.100968 |        0 |                0 |
| no preamble: (scheduled) | slowest            |      0.449321 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (scheduled <= "2019-01-01"))
#+END_SRC

#+RESULTS:
| Form                                     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|------------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (scheduled <= "2019-01-01")    | 4.13               |      0.111067 |        0 |                0 |
| no preamble: (scheduled <= "2019-01-01") | slowest            |      0.458726 |        0 |                0 |

*** =tags=

If tag inheritance is enabled, we have to check tags on every heading.  When it's disabled, we can search directly to headings with the given tags.

#+BEGIN_SRC elisp
  (let ((org-use-tag-inheritance t))
    (org-ql-preamble-bench :times 1
                           :file "~/org/inbox.org"
                           :query (tags "Emacs")))
#+END_SRC

#+RESULTS:
| Form                        | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-----------------------------+--------------------+---------------+----------+------------------|
| no preamble: (tags "Emacs") | 1.01               |      1.899647 |        0 |                0 |
| preamble: (tags "Emacs")    | slowest            |      1.921799 |        0 |                0 |

#+BEGIN_SRC elisp
  (let ((org-use-tag-inheritance nil))
    (org-ql-preamble-bench :times 1
                           :file "~/org/inbox.org"
                           :query (tags "Emacs")))
#+END_SRC

#+RESULTS:
| Form                        | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-----------------------------+--------------------+---------------+----------+------------------|
| preamble: (tags "Emacs")    | 2.08               |      0.274555 |        0 |                0 |
| no preamble: (tags "Emacs") | slowest            |      0.570116 |        0 |                0 |

*** ~ts~

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts))
#+END_SRC

#+RESULTS:
| Form              | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-------------------+--------------------+---------------+----------+------------------|
| preamble: (ts)    | 1.13               |      0.475646 |        0 |                0 |
| no preamble: (ts) | slowest            |      0.535950 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts :from "2019-01-01"))
#+END_SRC

#+RESULTS:
| Form                                 | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------+--------------------+---------------+----------+------------------|
| no preamble: (ts :from "2019-01-01") | 1.11               |      0.537445 |        0 |                0 |
| preamble: (ts :from "2019-01-01")    | slowest            |      0.594534 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts :from "2017-01-01"))
#+END_SRC

#+RESULTS:
| Form                                 | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------+--------------------+---------------+----------+------------------|
| no preamble: (ts :from "2017-01-01") | 1.13               |      0.526891 |        0 |                0 |
| preamble: (ts :from "2017-01-01")    | slowest            |      0.594360 |        0 |                0 |

Not sure why that one is slower with preamble.

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 10
                         :query (ts :from "2017-01-01"))
#+END_SRC

#+RESULTS:
| Form                                 | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------+--------------------+---------------+----------+------------------|
| no preamble: (ts :from "2017-01-01") | 1.04               |      0.025688 |        0 |                0 |
| preamble: (ts :from "2017-01-01")    | slowest            |      0.026642 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts :to "2010-01-01"))
#+END_SRC

#+RESULTS:
| Form                               | x faster than next | Total runtime | # of GCs | Total GC runtime |
|------------------------------------+--------------------+---------------+----------+------------------|
| no preamble: (ts :to "2010-01-01") | 1.10               |      0.538603 |        0 |                0 |
| preamble: (ts :to "2010-01-01")    | slowest            |      0.593466 |        0 |                0 |

*** ~ts-active~

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts-a))
#+END_SRC

#+RESULTS:
| Form                | x faster than next | Total runtime | # of GCs | Total GC runtime |
|---------------------+--------------------+---------------+----------+------------------|
| preamble: (ts-a)    | 4.77               |      0.071489 |        0 |                0 |
| no preamble: (ts-a) | slowest            |      0.340896 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts-a :from "2017-07-06"))
#+END_SRC

#+RESULTS:
| Form                                   | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (ts-a :from "2017-07-06")    | 1.78               |      0.188369 |        0 |                0 |
| no preamble: (ts-a :from "2017-07-06") | slowest            |      0.335975 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts-a :to "2017-07-06"))
#+END_SRC

#+RESULTS:
| Form                                 | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (ts-a :to "2017-07-06")    | 4.64               |      0.075307 |        0 |                0 |
| no preamble: (ts-a :to "2017-07-06") | slowest            |      0.349445 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts-a :on "2017-07-06"))
#+END_SRC

#+RESULTS:
| Form                                 | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------+--------------------+---------------+----------+------------------|
| preamble: (ts-a :on "2017-07-06")    | 4.33               |      0.076075 |        0 |                0 |
| no preamble: (ts-a :on "2017-07-06") | slowest            |      0.329106 |        0 |                0 |

*** ~ts-inactive~

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts-i))
#+END_SRC

#+RESULTS:
| Form                | x faster than next | Total runtime | # of GCs | Total GC runtime |
|---------------------+--------------------+---------------+----------+------------------|
| preamble: (ts-i)    | 1.21               |      0.459152 |        0 |                0 |
| no preamble: (ts-i) | slowest            |      0.555632 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts-i :from "2019-07-06"))
#+END_SRC

#+RESULTS:
| Form                                   | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------------------------------------+--------------------+---------------+----------+------------------|
| no preamble: (ts-i :from "2019-07-06") | 1.09               |      0.531976 |        0 |                0 |
| preamble: (ts-i :from "2019-07-06")    | slowest            |      0.579745 |        0 |                0 |

#+BEGIN_SRC elisp
  (org-ql-preamble-bench :times 1
                         :file "~/org/inbox.org"
                         :query (ts-i :to "2019-07-06"))
#+END_SRC

#+RESULTS:
| Form                                 | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------------------------------+--------------------+---------------+----------+------------------|
| no preamble: (ts-i :to "2019-07-06") | 1.34               |      0.553428 |        0 |                0 |
| preamble: (ts-i :to "2019-07-06")    | slowest            |      0.743881 |        0 |                0 |

** with/without ts.el

[2019-08-11 Sun 15:39]  These results seem to show a minor performance improvement by using ~ts~, and the code is simpler.

#+BEGIN_SRC elisp
  ;; (require 'ts)

  (org-ql--defpred ts-ts (&key from to _on)
    ;; The underscore before `on' prevents "unused lexical variable" warnings, because we
    ;; pre-process that argument in a macro before this function is called.
    "Return non-nil if current entry has a timestamp in given period.
  If no arguments are specified, return non-nil if entry has any
  timestamp.

  If FROM, return non-nil if entry has a timestamp on or after
  FROM.

  If TO, return non-nil if entry has a timestamp on or before TO.

  If ON, return non-nil if entry has a timestamp on date ON.

  FROM, TO, and ON should be strings parseable by
  `parse-time-string' but may omit the time value."
    ;; TODO: DRY this with the clocked predicate.
    ;; NOTE: FROM and TO are actually expected to be Unix timestamps.  The docstring is written
    ;; for end users, for which the arguments are pre-processed by `org-ql-select'.
    ;; FIXME: This assumes every "clocked" entry is a range.  Unclosed clock entries are not handled.
    (cl-macrolet ((next-timestamp ()
                                  `(when (re-search-forward org-element--timestamp-regexp end-pos t)
                                     (ts-parse-org (match-string 0))))
                  (test-timestamps (pred-form)
                                   `(cl-loop for next-ts = (next-timestamp)
                                             while next-ts
                                             thereis ,pred-form)))
      (save-excursion
        (let ((end-pos (org-entry-end-position)))
          (cond ((not (or from to)) (re-search-forward org-element--timestamp-regexp end-pos t))
                ((and from to) (test-timestamps (and (ts<= from next-ts)
                                                     (ts<= next-ts to))))
                (from (test-timestamps (ts<= from next-ts)))
                (to (test-timestamps (ts<= next-ts to))))))))

#+END_SRC

*** Without timestamp argument

#+BEGIN_SRC elisp
  (bench-multi-lexical :times 1 :ensure-equal t
    :forms (("old ts" (org-ql "~/org/inbox.org"
                        (ts)))
            ("ts.el ts" (org-ql "~/org/inbox.org"
                          (ts-ts)))))
#+END_SRC

#+RESULTS:
| Form     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------+--------------------+---------------+----------+------------------|
| ts.el ts | 1.14               |      2.251801 |        0 |                0 |
| old ts   | slowest            |      2.560280 |        0 |                0 |

#+BEGIN_SRC elisp
  (bench-multi-lexical :times 20 :ensure-equal t
    :forms (("old ts" (org-ql "~/src/emacs/org-ql/tests/data.org"
                        (ts)))
            ("ts.el ts" (org-ql "~/src/emacs/org-ql/tests/data.org"
                          (ts-ts)))))
#+END_SRC

#+RESULTS:
| Form     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------+--------------------+---------------+----------+------------------|
| ts.el ts | 1.05               |      0.103714 |        0 |                0 |
| old ts   | slowest            |      0.108663 |        0 |                0 |

*** :from

#+BEGIN_SRC elisp
  (bench-multi-lexical :times 1 :ensure-equal t
    :forms (("old ts" (org-ql "~/org/inbox.org"
                        (ts :from "2017-01-01")))
            ("ts.el ts" (org-ql "~/org/inbox.org"
                          (ts-ts :from "2017-01-01")))))
#+END_SRC

#+RESULTS:
| Form     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------+--------------------+---------------+----------+------------------|
| ts.el ts | 1.32               |      1.299966 |        0 |                0 |
| old ts   | slowest            |      1.713027 |        0 |                0 |

*** :to

#+BEGIN_SRC elisp
  (bench-multi-lexical :times 1 :ensure-equal t
    :forms (("old ts" (org-ql "~/org/inbox.org"
                        (ts :to "2019-01-01")))
            ("ts.el ts" (org-ql "~/org/inbox.org"
                          (ts-ts :to "2019-01-01")))))
#+END_SRC

#+RESULTS:
| Form     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------+--------------------+---------------+----------+------------------|
| ts.el ts | 1.01               |      1.300084 |        0 |                0 |
| old ts   | slowest            |      1.312208 |        0 |                0 |

*** :on

#+BEGIN_SRC elisp
  (bench-multi-lexical :times 1 :ensure-equal t
    :forms (("old ts" (org-ql "~/org/inbox.org"
                        (ts :on "2019-05-14")))
            ("ts.el ts" (org-ql "~/org/inbox.org"
                          (ts-ts :on "2019-05-14")))))
#+END_SRC

#+RESULTS:
| Form     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------+--------------------+---------------+----------+------------------|
| ts.el ts | 1.17               |      0.557281 |        0 |                0 |
| old ts   | slowest            |      0.652149 |        0 |                0 |

** Using =org-element-parse-buffer=

This basically works, as a very basic kind of agenda view, but we can already see that it's much slower (at least, for single-day views) because =org-element-parse-buffer= is slow compared to the agenda code.

[2018-05-10 Thu 15:03]  *Note:* This is the old, much slower code that used =org-element-parse-buffer=.

*** Macro

#+BEGIN_SRC elisp
  (defmacro elp-profile (times prefixes &rest body)
    (declare (indent defun))
    (let ((prefixes (append '(org- string- s- buffer- append delq map
                                   list car save- outline- delete-dups
                                   sort line- nth concat char-to-string
                                   rx- goto- when search- re-)
                            prefixes)))
      `(let (output)
         (dolist (prefix ',prefixes)
           (elp-instrument-package (symbol-name prefix)))
         (dotimes (x ,times)
           ,@body)
         (elp-results)
         (elp-restore-all)
         (point-min)
         (forward-line 20)
         (delete-region (point) (point-max))
         (setq output (buffer-substring-no-properties (point-min) (point-max)))
         (kill-buffer)
         (delete-window)
         (let ((rows (s-lines output)))
           (append (list (list "Function" "Times called" "Total time" "Average time")
                         'hline)
                   (cl-loop for row in rows
                            collect (s-split (rx (1+ space)) row 'omit-nulls)))))))
#+END_SRC

[2018-05-09 Wed 17:31]  *Note*: I seem to have misplaced the =org-agenda-ng--test-agenda-today= function I used in these tests.

*** ng

#+BEGIN_SRC elisp
  (elp-profile 1 (org-agenda-ng--test-agenda-today))
#+END_SRC

#+RESULTS:
#+begin_example
org-element--parse-elements                                   5832        18.501891926  0.0031724780
mapcar                                                        98          6.3412930759  0.0647070722
org-agenda-ng--test-agenda-today                              1           6.30112088    6.30112088
org-agenda-ng--agenda-multi                                   1           6.301086333   6.301086333
org-agenda-ng--get-entries                                    8           6.249823971   0.7812279963
mapc                                                          2803        5.9078545849  0.0021076898
org-element-parse-buffer                                      8           4.796204625   0.5995255781
org-element--current-element                                  6557        3.7164850469  0.0005667965
org-element-headline-parser                                   6557        3.5548915590  0.0005421521
org-end-of-subtree                                            6557        1.3663438270  0.0002083794
org-agenda-ng--filter-tree                                    8           1.3661297829  0.1707662228
org-element-map                                               8           1.365995685   0.1707494606
line-end-position                                             9503        0.4900104040  5.156...e-05
org-at-heading-p                                              12389       0.4574876539  3.692...e-05
re-search-forward                                             20193       0.4566419319  2.261...e-05
outline-on-heading-p                                          19286       0.4559736580  2.364...e-05
org-outline-level                                             6775        0.3970647569  5.860...e-05
org-back-to-heading                                           6897        0.3689033409  5.348...e-05
outline-back-to-heading                                       6897        0.3587580889  5.201...e-05
line-beginning-position                                       11702       0.2887473860  2.467...e-05
#+end_example

*** orig

Make sure to kill any existing agenda buffers first.

#+BEGIN_SRC elisp
  (elp-profile 1 (org-agenda-list nil nil 'week))
#+END_SRC

#+RESULTS:
#+begin_example
org-agenda-list                                               1           1.717467214   1.717467214
org-agenda-get-day-entries                                    7           1.124906724   0.1607009605
org-agenda-get-scheduled                                      7           0.9354116170  0.1336302310
org-get-tags-at                                               62          0.5598817790  0.0090303512
org-up-heading-safe                                           262         0.5531687240  0.0021113310
org-back-to-heading                                           2646        0.5316139889  0.0002009123
org-agenda-finalize                                           1           0.485185818   0.485185818
org-agenda-skip                                               749         0.4059732339  0.0005420203
org-at-planning-p                                             921         0.2137141959  0.0002320458
org-is-habit-p                                                574         0.1953095319  0.0003402605
org-entry-get                                                 579         0.195275877   0.0003372640
org--property-local-values                                    574         0.1899641870  0.0003309480
org-get-property-block                                        574         0.1789142650  0.0003116973
re-search-backward                                            3586        0.1769497870  4.934...e-05
org-inlinetask-in-task-p                                      1495        0.1639709820  0.0001096795
outline-back-to-heading                                       2646        0.1387423440  5.243...e-05
re-search-forward                                             3177        0.1353034909  4.258...e-05
org-agenda-get-deadlines                                      7           0.1341101260  0.0191585894
line-beginning-position                                       2041        0.1000089939  4.899...e-05
org-get-todo-state                                            749         0.070545971   9.418...e-05
org-agenda-prepare                                            1           0.052788647   0.052788647
org-agenda-prepare-buffers                                    1           0.050402675   0.050402675
org-agenda-get-timestamps                                     7           0.0428353419  0.0061193345
org-agenda--timestamp-to-absolute                             1498        0.0414448099  2.766...e-05
org-time-string-to-absolute                                   1498        0.0385546620  2.573...e-05
org-agenda-finalize-entries                                   7           0.03400573    0.0048579614
org-super-agenda--finalize-entries                            7           0.0339819269  0.0048545609
org-outline-level                                             505         0.0335176070  6.637...e-05
org-super-agenda--group-items                                 7           0.0268096709  0.0038299529
org-super-agenda--group-dispatch                              84          0.024216379   0.0002882902
org-parse-time-string                                         1572        0.0211881319  1.347...e-05
org-closest-date                                              749         0.0178231690  2.379...e-05
org-before-first-heading-p                                    578         0.0142076310  2.458...e-05
org-refresh-category-properties                               1           0.013815263   0.013815263
org-in-src-block-p                                            753         0.0135235599  1.795...e-05
org-refresh-stats-properties                                  1           0.012230309   0.012230309
org-habit-parse-todo                                          5           0.0121474539  0.0024294907
org-get-limited-outline-regexp                                1502        0.0100960829  6.721...e-06
mapcar                                                        309         0.009495786   3.073...e-05
org-super-agenda--group-habit                                 7           0.009318616   0.0013312308
string-match                                                  6253        0.0090207890  1.442...e-06
org-super-agenda--group-dispatch-and                          7           0.0067444     0.0009634857
org-agenda-get-blocks                                         7           0.0060974240  0.0008710605
outline-on-heading-p                                          2655        0.0060741090  2.287...e-06
org-agenda-get-sexps                                          7           0.0057924309  0.0008274901
org-super-agenda--group-regexp                                7           0.005592509   0.0007989298
org-refresh-properties                                        2           0.005461118   0.002730559
org-super-agenda--get-item-entry                              31          0.004812809   0.0001552519
org-agenda-align-tags                                         1           0.004677699   0.004677699
org-set-regexps-and-options                                   1           0.004533983   0.004533983
org--setup-collect-keywords                                   2           0.004499168   0.002249584
org-agenda-format-item                                        31          0.0039898020  0.0001287032
org-date-to-gregorian                                         420         0.0039648049  9.440...e-06
org-agenda-highlight-todo                                     31          0.0039479179  0.0001273521
string-to-number                                              8202        0.0031924329  3.892...e-07
org-end-of-subtree                                            12          0.003047285   0.0002539404
org-super-agenda--group-tag                                   35          0.002941374   8.403...e-05
org-inlinetask-outline-regexp                                 1495        0.002887697   1.931...e-06
org-get-wdays                                                 749         0.0027946290  3.731...e-06
org-refresh-effort-properties                                 1           0.002750279   0.002750279
org-entry-beginning-position                                  31          0.002345909   7.567...e-05
sort                                                          25          0.0021000059  8.400...e-05
mapc                                                          445         0.0018132390  4.074...e-06
line-end-position                                             72          0.00177188    2.460...e-05
mapconcat                                                     341         0.0016574119  4.860...e-06
org-entries-lessp                                             60          0.0016203450  2.700...e-05
concat                                                        336         0.0016188640  4.818...e-06
org-habit-insert-consistency-graphs                           1           0.001368346   0.001368346
org-add-props                                                 137         0.001291149   9.424...e-06
org-element-at-point                                          1           0.001266403   0.001266403
org-agenda-new-marker                                         62          0.001253987   2.022...e-05
outline-next-heading                                          42          0.001123596   2.675...e-05
org-split-string                                              148         0.0011049120  7.465...e-06
org-element--parse-to                                         1           0.001074227   0.001074227
org-get-scheduled-time                                        5           0.0010200300  0.000204006
org-super-agenda--group-todo                                  21          0.001003225   4.777...e-05
org-time-string-to-time                                       69          0.0009947770  1.441...e-05
org-element--current-element                                  4           0.0009597410  0.0002399352
org-entry-properties                                          5           0.0008807149  0.0001761429
org-at-date-range-p                                           172         0.0008412679  4.891...e-06
org-agenda-mode                                               1           0.000787997   0.000787997
org-super-agenda--group-log                                   14          0.0007874939  5.624...e-05
buffer-substring                                              31          0.0007650360  2.467...e-05
org-heading-components                                        9           0.00071644    7.960...e-05
org-entry-end-position                                        31          0.0007046580  2.273...e-05
string-prefix-p                                               789         0.0006732279  8.532...e-07
org-agenda-skip-eval                                          1498        0.0006586869  4.397...e-07
org-get-repeat                                                5           0.000656577   0.0001313153
org-activate-bracket-links                                    3           0.000648713   0.0002162376
org-agenda-fix-displayed-tags                                 31          0.000636446   2.053...e-05
s-join                                                        252         0.0006324150  2.509...e-06
org-element-keyword-parser                                    4           0.000530311   0.0001325777
org-in-commented-heading-p                                    4           0.00051077    0.0001276925
buffer-substring-no-properties                                308         0.0004763350  1.546...e-06
org-habit-build-graph                                         5           0.0004689199  9.3784e-05
car                                                           1626        0.0004469320  2.748...e-07
org-activate-plain-links                                      3           0.000382191   0.000127397
org-super-agenda--transform-groups                            7           0.0003747289  5.353...e-05
org-get-priority                                              26          0.0003598600  1.384...e-05
org-agenda-fontify-priorities                                 1           0.000357076   0.000357076
org-super-agenda--group-priority                              14          0.00035224    2.516e-05
org-agenda-prepare-window                                     1           0.00033704    0.00033704
org-find-text-property-in-string                              316         0.0003191880  1.010...e-06
org-not-nil                                                   821         0.0003016530  3.674...e-07
list                                                          940         0.0002995150  3.186...e-07
org-super-agenda--group-time-grid                             7           0.0002954999  4.221...e-05
org-agenda-format-date-aligned                                7           0.0002794120  3.991...e-05
org-super-agenda--get-tags                                    103         0.0002654610  2.577...e-06
delq                                                          513         0.0002634709  5.135...e-07
org-replace-escapes                                           5           0.0002382050  4.7641e-05
car-safe                                                      911         0.0002314199  2.540...e-07
org-today                                                     48          0.0002232539  4.651...e-06
org-get-time-of-day                                           19          0.000219044   1.152...e-05
org-days-to-iso-week                                          9           0.0002125040  2.361...e-05
append                                                        433         0.0002083290  4.811...e-07
org-get-category                                              31          0.000198604   6.406...e-06
string-match-p                                                31          0.0001787880  5.767...e-06
org-agenda-today-p                                            21          0.0001658670  7.898...e-06
org-super-agenda--get-marker                                  54          0.0001629399  3.017...e-06
delete-dups                                                   111         0.0001343219  1.210...e-06
org-at-heading-p                                              9           0.000128871   1.4319e-05
string-equal                                                  361         0.0001168969  3.238...e-07
org-super-agenda--make-agenda-header                          13          0.0001151760  8.859...e-06
org-add-prop-inherited                                        142         0.0001142290  8.044...e-07
org-check-agenda-file                                         8           0.000110448   1.3806e-05
org-agenda-get-day-face                                       7           0.000110437   1.577...e-05
org-get-agenda-file-buffer                                    8           0.000107414   1.342675e-05
org-agenda-files                                              3           0.000102652   3.421...e-05
org-downcase-keep-props                                       90          0.0001008230  1.120...e-06
org-super-agenda--get-priority-cookie                         15          9.460...e-05  6.307e-06
org-get-todo-face                                             31          9.423...e-05  3.039...e-06
org-find-base-buffer-visiting                                 8           8.7492e-05    1.09365e-05
org-habit-get-faces                                           140         8.080...e-05  5.771...e-07
org-remove-uninherited-tags                                   176         7.580...e-05  4.307...e-07
goto-char                                                     31          7.1014e-05    2.290...e-06
org-super-agenda--transform-group-order                       7           6.5543e-05    9.363...e-06
org-agenda-add-time-grid-maybe                                7           5.703...e-05  8.147...e-06
org-compile-prefix-format                                     1           5.2164e-05    5.2164e-05
org-link-unescape                                             4           4.4823e-05    1.120575e-05
listp                                                         133         3.960...e-05  2.977...e-07
org-habit-duration-to-days                                    5           3.9446e-05    7.8892e-06
org-make-options-regexp                                       1           3.6054e-05    3.6054e-05
org-agenda-mark-header-line                                   1           3.3066e-05    3.3066e-05
org-element--collect-affiliated-keywords                      4           3.2661e-05    8.16525e-06
org-habit-get-priority                                        5           2.9857e-05    5.9714e-06
org-agenda-get-category-icon                                  31          2.673...e-05  8.624...e-07
org-face-from-face-or-color                                   61          2.1803e-05    3.574...e-07
org-reduced-level                                             40          2.1692e-05    5.423e-07
org-link-get-parameter                                        26          2.062...e-05  7.930...e-07
buffer-name                                                   57          1.9875e-05    3.486...e-07
org-defkey                                                    5           1.9835e-05    3.966...e-06
string-to-char                                                56          1.929...e-05  3.446...e-07
org-string-nw-p                                               5           1.7894e-05    3.578...e-06
s-wrap                                                        7           1.6955e-05    2.422...e-06
org-link-expand-abbrev                                        2           1.5491e-05    7.7455e-06
map-keymap                                                    2           1.4987e-05    7.4935e-06
search-forward                                                7           1.403...e-05  2.004...e-06
string-lessp                                                  41          1.394...e-05  3.400...e-07
org-agenda-reset-markers                                      1           1.3927e-05    1.3927e-05
org-agenda-deadline-face                                      7           1.2592e-05    1.798...e-06
search-backward                                               5           1.1253e-05    2.2506e-06
org-agenda-span-name                                          5           9.758e-06     1.9516e-06
outline-previous-heading                                      1           8.678e-06     8.678e-06
buffer-modified-p                                             10          4.618e-06     4.617...e-07
buffer-file-name                                              13          4.608...e-06  3.544...e-07
buffer-live-p                                                 11          4.473e-06     4.066...e-07
buffer-local-value                                            10          4.278e-06     4.278...e-07
buffer-base-buffer                                            10          3.858e-06     3.858...e-07
org-agenda-set-mode-name                                      1           3.118e-06     3.118e-06
org-remove-flyspell-overlays-in                               4           2.926...e-06  7.315...e-07
buffer-size                                                   8           2.791...e-06  3.489...e-07
org-key                                                       5           2.707e-06     5.414e-07
org-unbracket-string                                          1           2.692e-06     2.692e-06
save-place-to-alist                                           1           2.276e-06     2.276e-06
org-element--cache-put                                        4           1.907e-06     4.7675e-07
org-property-inherit-p                                        2           1.414e-06     7.07e-07
org-agenda-ndays-to-span                                      2           1.293...e-06  6.465...e-07
maphash                                                       1           1.171e-06     1.171e-06
org-set-sorting-strategy                                      1           1.165e-06     1.165e-06
org-file-menu-entry                                           1           1.12e-06      1.12e-06
string-width                                                  2           1.093e-06     5.465e-07
org-time-stamp-format                                         1           1.054e-06     1.054e-06
org-agenda-fit-window-to-buffer                               1           9.31e-07      9.31e-07
org-font-lock-add-tag-faces                                   1           9.13e-07      9.13e-07
org-agenda-span-to-ndays                                      1           8.3e-07       8.3e-07
org-element-property                                          2           8.12e-07      4.06e-07
org-agenda-mark-clocking-task                                 1           8.03e-07      8.03e-07
org-tag-alist-to-groups                                       1           7.55e-07      7.55e-07
org-element-type                                              1           6.14e-07      6.14e-07
org-agenda-use-sticky-p                                       1           4.73e-07      4.73e-07
mapatoms                                                      1           0             0.0
#+end_example

*** Profile org-element-map

#+BEGIN_SRC elisp
  (elp-profile 1 (with-current-buffer (find-buffer-visiting "~/org/main.org")
                   (org-element-parse-buffer 'headline)))
#+END_SRC

#+RESULTS:
#+begin_example
org-element--parse-elements                                   1002        4.1612395469  0.0041529336
org-element-parse-buffer                                      1           0.859981956   0.859981956
org-element--current-element                                  1225        0.8236391779  0.0006723585
org-element-headline-parser                                   1225        0.7952382879  0.0006491741
org-end-of-subtree                                            1225        0.5557043549  0.0004536362
line-end-position                                             1995        0.0751104350  3.764...e-05
re-search-forward                                             3743        0.0516547359  1.380...e-05
org-outline-level                                             1225        0.0477962079  3.901...e-05
org-back-to-heading                                           1225        0.0469223529  3.830...e-05
outline-back-to-heading                                       1225        0.0450936199  3.681...e-05
org-element--get-node-properties                              1225        0.0434517140  3.547...e-05
line-beginning-position                                       2003        0.0334129610  1.668...e-05
org-element--get-time-properties                              1225        0.0306394040  2.501...e-05
org-get-limited-outline-regexp                                2072        0.0140703559  6.790...e-06
org-element-timestamp-parser                                  249         0.0124751890  5.010...e-05
outline-next-heading                                          847         0.0115396399  1.362...e-05
org-at-heading-p                                              2227        0.0109473260  4.915...e-06
outline-on-heading-p                                          3452        0.0101107139  2.928...e-06
string-match                                                  4594        0.0064121759  1.395...e-06
mapcar                                                        30          0.0041008740  0.0001366958
#+end_example

** Profiling position-based

*** Macro

#+BEGIN_SRC elisp
  (defmacro elp-profile (times &rest body)
    (declare (indent defun))
    `(let ((prefixes '("org-" "string-" "s-" "buffer-" "append" "delq" "map"
                       "list" "car" "save-" "outline-" "delete-dups"
                       "sort" "line-" "nth" "concat" "char-to-string"
                       "rx-" "goto-" "when" "search-" "re-"))
           output)
       (dolist (prefix prefixes)
         (elp-instrument-package prefix))
       (dotimes (x ,times)
         ,@body)
       (elp-results)
       (elp-restore-all)
       (point-min)
       (forward-line 20)
       (delete-region (point) (point-max))
       (setq output (buffer-substring-no-properties (point-min) (point-max)))
       (kill-buffer)
       (delete-window)
       output))
#+END_SRC


*** orig

Make sure to kill any existing agenda buffers first.

#+BEGIN_SRC elisp
  (elp-profile 1 (org-agenda-list nil nil 'week))
#+END_SRC

#+RESULTS:
#+begin_example
org-agenda-list                                               1           9.693596196   9.693596196
org-agenda-get-day-entries                                    56          8.630330659   0.1541130474
org-agenda-get-scheduled                                      56          6.6207980570  0.1182285367
org-is-habit-p                                                2792        2.2907458449  0.0008204677
org-entry-get                                                 2798        2.287390186   0.0008175090
org-agenda--timestamp-to-absolute                             7708        2.0970420100  0.0002720604
org-agenda-get-deadlines                                      56          1.6941886389  0.0302533685
org-at-planning-p                                             4399        1.3993312159  0.0003181021
org--property-local-values                                    2793        1.2699226760  0.0004546805
org-get-property-block                                        2794        1.2182695930  0.0004360306
org-time-string-to-absolute                                   7708        1.1513844880  0.0001493752
org-inlinetask-in-task-p                                      6969        1.139932302   0.0001635718
org-parse-time-string                                         7880        1.0635759220  0.0001349715
org-closest-date                                              3864        1.0383435800  0.0002687224
re-search-forward                                             15199       0.9607921779  6.321...e-05
org-back-to-heading                                           12667       0.8564486210  6.761...e-05
outline-back-to-heading                                       12667       0.8362207570  6.601...e-05
line-beginning-position                                       10333       0.7998346869  7.740...e-05
org-agenda-format-item                                        279         0.7552402350  0.0027069542
re-search-backward                                            16726       0.5694224969  3.404...e-05
#+end_example

*** ng-funcall

#+BEGIN_SRC elisp
  (elp-profile 5 (org-agenda-ng--test-agenda-today))
#+END_SRC

#+RESULTS:
#+begin_example
mapcar                                                        121         0.1296645480  0.0010716078
org-agenda-ng--test-agenda-today                              5           0.086714029   0.0173428058
org-agenda-ng--agenda                                         5           0.086584611   0.0173169222
org-agenda-ng--format-element                                 75          0.0307461019  0.0004099480
org-agenda-ng--filter-buffer                                  5           0.027136826   0.0054273652
org-agenda-ng--date-p                                         455         0.0213037090  4.682...e-05
org-element-headline-parser                                   75          0.016251755   0.0002166900
org-get-tags-at                                               75          0.008959605   0.0001194614
org-agenda-ng--add-faces                                      75          0.0072381410  9.650...e-05
org-element-timestamp-interpreter                             150         0.0069832960  4.655...e-05
org-entry-get                                                 290         0.0061220340  2.111...e-05
org-up-heading-safe                                           210         0.0057036860  2.716...e-05
org-agenda-finalize-entries                                   5           0.005372899   0.0010745798
org-element-timestamp-parser                                  150         0.0050518689  3.367...e-05
org-entry-properties                                          290         0.0049517209  1.707...e-05
org-agenda-ng--add-deadline-face                              75          0.0039273909  5.236...e-05
org-element--get-time-properties                              75          0.0039059429  5.207...e-05
org-back-to-heading                                           725         0.0037793259  5.212...e-06
org-parse-time-string                                         300         0.0032196259  1.073...e-05
org-agenda-ng--add-scheduled-face                             75          0.0031410580  4.188...e-05
#+end_example

*** ng-flet

#+BEGIN_SRC elisp
  (elp-profile 5 (org-agenda-ng--test-agenda-today))
#+END_SRC

#+RESULTS:
#+begin_example
mapcar                                                        121         0.1292609089  0.0010682719
org-agenda-ng--test-agenda-today                              5           0.0860146149  0.017202923
org-agenda-ng--agenda                                         5           0.0858901769  0.0171780353
org-agenda-ng--format-element                                 75          0.0308815090  0.0004117534
org-agenda-ng--filter-buffer                                  5           0.026709027   0.0053418054
org-agenda-ng--date-p                                         455         0.0210552310  4.627...e-05
org-element-headline-parser                                   75          0.016209908   0.0002161321
org-get-tags-at                                               75          0.008953666   0.0001193822
org-agenda-ng--add-faces                                      75          0.0072834109  9.711...e-05
org-element-timestamp-interpreter                             150         0.0068781430  4.585...e-05
org-entry-get                                                 290         0.0060815609  2.097...e-05
org-up-heading-safe                                           210         0.005708647   2.718...e-05
org-agenda-finalize-entries                                   5           0.005201221   0.0010402442
org-element-timestamp-parser                                  150         0.005191617   3.461078e-05
org-entry-properties                                          290         0.0048787450  1.682...e-05
org-element--get-time-properties                              75          0.004112675   5.483...e-05
org-agenda-ng--add-deadline-face                              75          0.0039314910  5.241...e-05
org-back-to-heading                                           725         0.0037559990  5.180...e-06
org-agenda-ng--add-scheduled-face                             75          0.0031766149  4.235...e-05
org-parse-time-string                                         300         0.0031740200  1.058...e-05
#+end_example

** Profiling =org-trust-scanner-tags=

[2018-05-10 Thu 12:59]  Turned on =org-trust-scanner-tags=, going to try profiling again:

#+BEGIN_SRC elisp
  ;; (elp-profile 1 nil (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
  ;;                      (tags "world")))

  (elp-profile 10 nil (org-agenda-ng org-agenda-files
                       (tags "Emacs")))
#+END_SRC

#+RESULTS:
| Function                                  | Times called |   Total time | Average time |
|-------------------------------------------+--------------+--------------+--------------|
| org-agenda-ng--agenda                     |           10 | 44.092598282 | 4.4092598282 |
| mapcar                                    |          282 | 40.234516707 | 0.1426755911 |
| org-agenda-ng--filter-buffer              |           80 | 26.895492471 | 0.3361936558 |
| org-element-headline-parser               |         3980 | 10.387614362 | 0.0026099533 |
| org-agenda-finalize-entries               |           10 |  9.194458252 | 0.9194458252 |
| org-agenda-ng--tags-p                     |        70250 | 8.1897379849 | 0.0001165799 |
| org-agenda-ng--format-element             |         3980 | 6.5944325679 | 0.0016568926 |
| outline-next-heading                      |        70320 | 6.1190180490 | 8.701...e-05 |
| re-search-forward                         |        97050 | 5.8706467829 | 6.049...e-05 |
| org-get-tags-at                           |        74230 | 5.4078158059 | 7.285...e-05 |
| org-super-agenda--filter-finalize-entries |           10 | 5.2320123400 | 0.5232012340 |
| org-super-agenda--group-items             |           10 | 5.1260959210 | 0.5126095921 |
| org-super-agenda--group-dispatch          |          130 |  5.119333624 | 0.0393794894 |
| sort                                      |           20 | 3.8204368569 | 0.1910218428 |
| org-element--parse-objects                |         6180 | 3.5386578929 | 0.0005725983 |
| org-is-habit-p                            |         5970 | 3.2497755920 | 0.0005443510 |
| org-entry-get                             |         5970 | 3.2347964049 | 0.0005418419 |
| org--property-local-values                |         5970 | 3.1796357319 | 0.0005326023 |
| org-get-property-block                    |         5970 | 3.0767919680 | 0.0005153755 |
| org-entries-lessp                         |        20020 | 2.6563960079 | 0.0001326871 |

Now trying again without it:

#+BEGIN_SRC elisp
  ;; (elp-profile 1 nil (org-agenda-ng "~/src/emacs/org-super-agenda/test/test.org"
  ;;                      (tags "world")))

  (elp-profile 10 nil (org-agenda-ng org-agenda-files
                       (tags "Emacs")))
#+END_SRC

#+RESULTS:
| Function                                  | Times called |   Total time | Average time |
|-------------------------------------------+--------------+--------------+--------------|
| mapcar                                    |         1791 | 57.096304538 | 0.0318795670 |
| org-agenda-ng--agenda                     |           10 | 54.232133506 | 5.4232133505 |
| org-agenda-ng--filter-buffer              |           80 | 30.065167040 | 0.3758145880 |
| org-get-tags-at                           |        74230 | 13.840202495 | 0.0001864502 |
| org-agenda-ng--format-element             |         3980 | 13.429297797 | 0.0033741954 |
| org-element-headline-parser               |         3980 | 12.771776652 | 0.0032089891 |
| org-agenda-finalize-entries               |           10 | 9.1439433990 | 0.9143943399 |
| org-agenda-ng--tags-p                     |        70250 | 9.0249653730 | 0.0001284692 |
| org-super-agenda--filter-finalize-entries |           10 |  7.300515859 | 0.7300515859 |
| outline-next-heading                      |        70320 | 7.2384435649 | 0.0001029357 |
| org-super-agenda--group-items             |           10 |  4.918585855 | 0.4918585855 |
| org-super-agenda--group-dispatch          |          130 | 4.9125893509 | 0.0377891488 |
| re-search-forward                         |       101020 | 4.6294823850 | 4.582...e-05 |
| org-up-heading-safe                       |         7370 | 4.4629885620 | 0.0006055615 |
| org-is-habit-p                            |         5960 | 4.2772351910 | 0.0007176569 |
| org-entry-get                             |         5960 | 4.2595350800 | 0.0007146870 |
| org-super-agenda--group-tag               |           50 | 3.8942044929 | 0.0778840898 |
| re-search-backward                        |        26150 | 3.3660083490 | 0.0001287192 |
| org--property-local-values                |         5960 | 3.1793476329 | 0.0005334475 |
| org-get-property-block                    |         5960 | 3.0662425979 | 0.0005144702 |

Wow, using =org-trust-scanner-tags= saves a /lot/ of time.

** Profiling flet across all agenda files

*** Without flet

#+BEGIN_SRC elisp
  (elp-profile 5 (org-agenda-ng--agenda
                  :files org-agenda-files
                  :pred (lambda ()
                          (and (org-agenda-ng--todo-p)
                               (or (org-agenda-ng--date-p :deadline '<= (org-today))
                                   (org-agenda-ng--date-p :scheduled '<= (org-today)))
                               (not (apply #'org-agenda-ng--todo-p org-done-keywords-for-agenda))))))
#+END_SRC

#+RESULTS:
#+begin_example
mapcar                                                        711         26.910164986  0.0378483333
org-agenda-ng--agenda                                         5           21.012501837  4.2025003674
org-agenda-ng--filter-buffer                                  40          13.751964650  0.3437991162
org-agenda-ng--todo-p                                         37080       5.8788306440  0.0001585445
org-agenda-ng--format-element                                 1180        4.5712275970  0.0038739216
org-get-todo-state                                            37080       4.1661659069  0.0001123561
org-agenda-ng--date-p                                         21595       4.1442710769  0.0001919088
org-entry-get                                                 22730       2.8275069239  0.0001243953
org-entry-properties                                          21595       2.6558403739  0.0001229840
outline-next-heading                                          34625       2.0894695999  6.034...e-05
org-element-headline-parser                                   1180        1.9110445780  0.0016195293
re-search-forward                                             42280       1.6994989150  4.019...e-05
org-agenda-ng--add-faces                                      1180        1.6172592580  0.0013705586
org-agenda-ng--add-scheduled-face                             1180        1.607386145   0.0013621916
org-get-tags-at                                               1180        1.1521010509  0.0009763568
org-back-to-heading                                           64530       1.1005834200  1.705...e-05
org-up-heading-safe                                           2360        1.0182265390  0.0004314519
outline-back-to-heading                                       64530       1.0086056729  1.563...e-05
org-parse-time-string                                         7560        0.8314918499  0.0001099856
org-time-string-to-absolute                                   3780        0.8277485280  0.0002189810
#+end_example


*** With flet

#+BEGIN_SRC elisp
  (elp-profile 5 (org-agenda-ng--agenda
                  :files org-agenda-files
                  :pred (lambda ()
                          (and (todo)
                               (or (date :deadline '<= (org-today))
                                   (date :scheduled '<= (org-today)))
                               (not (apply #'todo org-done-keywords-for-agenda))))))
#+END_SRC

#+RESULTS:
#+begin_example
mapcar                                                        711         25.608392569  0.0360174297
org-agenda-ng--agenda                                         5           24.019318793  4.8038637586
org-agenda-ng--filter-buffer                                  40          14.160293256  0.3540073313
org-agenda-ng--date-p                                         21595       4.2111783960  0.0001950071
org-agenda-finalize-entries                                   5           4.0930243110  0.8186048622
org-super-agenda--filter-finalize-entries                     5           3.937522006   0.7875044012
org-agenda-ng--todo-p                                         37080       3.5687476730  9.624...e-05
org-get-todo-state                                            37080       3.4737076600  9.368...e-05
outline-next-heading                                          34625       3.4689080650  0.0001001850
re-search-forward                                             42280       3.0743315830  7.271...e-05
org-agenda-ng--format-element                                 1180        2.9511605820  0.0025009835
org-element-headline-parser                                   1180        2.6757063699  0.0022675477
org-super-agenda--group-items                                 5           2.187362092   0.4374724183
org-super-agenda--group-dispatch                              70          2.184685662   0.0312097951
org-entry-get                                                 22730       2.0711872869  9.112...e-05
org-entry-properties                                          21595       1.8958912070  8.779...e-05
org-super-agenda--group-tag                                   25          1.8498977799  0.0739959111
org-element-timestamp-parser                                  3785        1.8234333229  0.0004817525
org-parse-time-string                                         7560        1.7121709579  0.0002264776
org-element--get-time-properties                              1180        1.1814058020  0.0010011913
#+end_example

** Profiling flet on a single file

This shows that the difference between them, if any, is so small as to be irrelevant.  The convenience and clarity are a big win.

*** Without flet

#+BEGIN_SRC elisp
  (elp-profile 5 (org-agenda-ng--agenda
                  :files "~/org/main.org"
                  :pred (lambda ()
                          (and (org-agenda-ng--todo-p)
                               (or (org-agenda-ng--date-p :deadline '<= (org-today))
                                   (org-agenda-ng--date-p :scheduled '<= (org-today)))
                               (not (apply #'org-agenda-ng--todo-p org-done-keywords-for-agenda))))))
#+END_SRC

#+RESULTS:
#+begin_example
mapcar                                                        526         3.7766218089  0.0071798893
org-agenda-ng--agenda                                         5           2.75718831    0.551437662
org-agenda-ng--filter-buffer                                  5           1.402551392   0.2805102784
org-agenda-ng--format-element                                 265         0.8864161399  0.0033449665
org-get-tags-at                                               265         0.7896260759  0.0029797210
org-up-heading-safe                                           1150        0.7589292910  0.0006599385
re-search-backward                                            3700        0.5956338739  0.0001609821
org-agenda-ng--todo-p                                         6690        0.5781650060  8.642...e-05
org-get-todo-state                                            6690        0.5603983020  8.376...e-05
org-agenda-ng--date-p                                         5940        0.5209897369  8.770...e-05
org-entry-get                                                 6195        0.4158440950  6.712...e-05
org-entry-properties                                          5940        0.3640524090  6.128...e-05
org-element-headline-parser                                   265         0.2810144710  0.0010604319
outline-next-heading                                          6195        0.2485497380  4.012...e-05
org-back-to-heading                                           14565       0.1957209180  1.343...e-05
re-search-forward                                             7850        0.1927130979  2.454...e-05
outline-back-to-heading                                       14565       0.1751091780  1.202...e-05
org-outline-level                                             2300        0.1680958539  7.308...e-05
org-agenda-finalize-entries                                   5           0.1610422239  0.0322084448
org-super-agenda--filter-finalize-entries                     5           0.132423043   0.0264846085
#+end_example


*** With flet

#+BEGIN_SRC elisp
  (elp-profile 5 (org-agenda-ng--agenda
                  :files "~/org/main.org"
                  :pred (lambda ()
                          (and (todo)
                               (or (date :deadline '<= (org-today))
                                   (date :scheduled '<= (org-today)))
                               (not (apply #'todo org-done-keywords-for-agenda))))))
#+END_SRC

#+RESULTS:
#+begin_example
mapcar                                                        526         3.7898506779  0.0072050393
org-agenda-ng--agenda                                         5           2.7695176850  0.5539035370
org-agenda-ng--filter-buffer                                  5           1.414347774   0.2828695548
org-agenda-ng--format-element                                 265         0.8871611419  0.0033477778
org-get-tags-at                                               265         0.7891641319  0.0029779778
org-up-heading-safe                                           1150        0.7581951110  0.0006593000
re-search-backward                                            3700        0.5948686769  0.0001607753
org-agenda-ng--todo-p                                         6690        0.5840980579  8.730...e-05
org-get-todo-state                                            6690        0.5666448919  8.470...e-05
org-agenda-ng--date-p                                         5940        0.5196037069  8.747...e-05
org-entry-get                                                 6195        0.4144106150  6.689...e-05
org-entry-properties                                          5940        0.3640680380  6.129...e-05
org-element-headline-parser                                   265         0.2810144920  0.0010604320
outline-next-heading                                          6195        0.2495287770  4.027...e-05
org-back-to-heading                                           14565       0.1959557380  1.345...e-05
re-search-forward                                             7850        0.1933439489  2.462...e-05
outline-back-to-heading                                       14565       0.1753121230  1.203...e-05
org-outline-level                                             2300        0.1676228200  7.287...e-05
org-agenda-finalize-entries                                   5           0.1607656930  0.0321531386
org-super-agenda--filter-finalize-entries                     5           0.1316961509  0.0263392301
#+end_example

** Profiling tags matching

*** ng

#+BEGIN_SRC elisp
  (elp-profile 1 nil
    (org-agenda-ng "~/org/main.org"
      (tags "computer")))
#+END_SRC

#+RESULTS:
| Function                       | Times called |   Total time | Average time |
|--------------------------------+--------------+--------------+--------------|
| mapcar                         |         4217 | 12.612716455 | 0.0029909216 |
| org-agenda-ng--agenda          |            1 |  9.721410651 |  9.721410651 |
| org-get-tags-at                |         1845 | 7.4793860389 | 0.0040538677 |
| org-up-heading-safe            |         9361 | 6.4622674019 | 0.0006903394 |
| re-search-backward             |        25001 | 5.3399866239 | 0.0002135909 |
| org-agenda-ng--filter-buffer   |            1 |  4.874598854 |  4.874598854 |
| org-agenda-ng--tags-p          |         1238 | 4.8067623430 | 0.0038826836 |
| org-agenda-ng--format-element  |          607 | 3.6325626609 | 0.0059844524 |
| org-outline-level              |        17484 | 1.0298924459 | 5.890...e-05 |
| org-add-props                  |         2074 | 0.8305549259 | 0.0004004604 |
| org-element-headline-parser    |          607 | 0.2092664829 | 0.0003447553 |
| org-back-to-heading            |        11813 | 0.1252112960 | 1.059...e-05 |
| outline-back-to-heading        |        11813 | 0.1100693780 | 9.317...e-06 |
| org-end-of-subtree             |          607 | 0.0721986340 | 0.0001189433 |
| outline-on-heading-p           |        11813 | 0.0675261030 | 5.716...e-06 |
| outline-next-heading           |         1239 | 0.0627980999 | 5.068...e-05 |
| re-search-forward              |         3273 | 0.0612446620 | 1.871...e-05 |
| org-agenda-finalize-entries    |            1 |  0.041846274 |  0.041846274 |
| buffer-substring-no-properties |         6329 | 0.0308716979 | 4.877...e-06 |
| line-end-position              |          903 | 0.0280484950 | 3.106...e-05 |

*** ng without inheritance

#+BEGIN_SRC elisp
  (elp-profile 1 nil
    (org-agenda-ng "~/org/main.org"
      (tags "computer")))
#+END_SRC

#+RESULTS:
| Function                       | Times called |   Total time | Average time |
|--------------------------------+--------------+--------------+--------------|
| mapcar                         |         4217 | 12.580246839 | 0.0029832219 |
| org-agenda-ng--agenda          |            1 |  8.777776059 |  8.777776059 |
| org-get-tags-at                |         1845 | 8.2853503299 | 0.0044907047 |
| org-up-heading-safe            |         9361 | 7.2710981889 | 0.0007767437 |
| re-search-backward             |        25001 | 5.3360082060 | 0.0002134317 |
| org-agenda-ng--filter-buffer   |            1 |  4.865602689 |  4.865602689 |
| org-agenda-ng--tags-p          |         1238 | 4.7983754310 | 0.0038759090 |
| org-agenda-ng--format-element  |          607 | 3.6273825100 | 0.0059759184 |
| org-outline-level              |        17484 | 1.0284417919 | 5.882...e-05 |
| org-back-to-heading            |        11813 | 0.9390534479 | 7.949...e-05 |
| org-split-string               |         4940 |  0.833825087 | 0.0001687905 |
| string-match                   |         9102 | 0.8231629100 | 9.043...e-05 |
| org-element-headline-parser    |          607 | 0.2034305819 | 0.0003351409 |
| outline-back-to-heading        |        11813 | 0.1096120189 | 9.278...e-06 |
| org-end-of-subtree             |          607 | 0.0710802559 | 0.0001171009 |
| outline-on-heading-p           |        11813 | 0.0670029359 | 5.671...e-06 |
| outline-next-heading           |         1239 | 0.0622323519 | 5.022...e-05 |
| re-search-forward              |         3273 | 0.0603102519 | 1.842...e-05 |
| org-agenda-finalize-entries    |            1 |  0.037286496 |  0.037286496 |
| buffer-substring-no-properties |         6329 | 0.0285818689 | 4.516...e-06 |


*** original

#+BEGIN_SRC elisp
  (elp-profile 1 nil
    (with-current-buffer "main.org"
(org-tags-view nil "computer")))
#+END_SRC

#+RESULTS:
| Function                    | Times called |   Total time | Average time |
|-----------------------------+--------------+--------------+--------------|
| org-tags-view               |            1 |  2.620578129 |  2.620578129 |
| org-scan-tags               |            1 |  1.448883817 |  1.448883817 |
| org-agenda-format-item      |          607 | 0.9273893060 | 0.0015278242 |
| org-add-props               |         2042 | 0.8877267209 | 0.0004347339 |
| org-agenda-finalize         |            1 |  0.144506782 |  0.144506782 |
| re-search-forward           |         2154 | 0.1367046650 | 6.346...e-05 |
| string-match                |         8742 | 0.1002517259 | 1.146...e-05 |
| org-get-priority            |          607 | 0.0961996220 | 0.0001584837 |
| org-agenda-align-tags       |            1 |  0.095166495 |  0.095166495 |
| org-agenda-prepare          |            1 |  0.081724472 |  0.081724472 |
| org-outline-level           |         1246 | 0.0771033170 | 6.188...e-05 |
| org-agenda-finalize-entries |            1 |  0.071707404 |  0.071707404 |
| org-agenda-prepare-buffers  |            1 |  0.057903921 |  0.057903921 |
| org-get-heading             |          607 | 0.0517784369 | 8.530...e-05 |
| mapcar                      |         3738 | 0.0418641110 | 1.119...e-05 |
| org-agenda-highlight-todo   |          607 | 0.0273123070 | 4.499...e-05 |
| mapconcat                   |          609 |  0.024743305 | 4.062...e-05 |
| sort                        |            2 |   0.02117069 |  0.010585345 |
| org-activate-plain-links    |          132 | 0.0203558980 | 0.0001542113 |
| org-activate-bracket-links  |           78 | 0.0198589680 | 0.0002546021 |

** More profiling

[2018-05-10 Thu 15:02]  I think these are decent improvements.

#+BEGIN_SRC elisp
  (elp-profile 1 nil (org-agenda-ng "~/org/main.org"
                        (or (habit)
                            (and (or (date '= (org-today))
                                     (deadline '<=)
                                     (scheduled '<= (org-today)))
                                 (not (apply #'todo org-done-keywords-for-agenda)))
                            (and (todo "DONE" "CANCELLED")
                                 (closed '= (org-today))))))
#+END_SRC

#+RESULTS:
| Function                      | Times called |   Total time | Average time |
|-------------------------------+--------------+--------------+--------------|
| mapcar                        |          164 | 1.5004585290 | 0.0091491373 |
| org-agenda-ng--agenda         |            1 |  1.348231247 |  1.348231247 |
| org-agenda-ng--filter-buffer  |            1 | 1.1391189879 | 1.1391189879 |
| org-agenda-ng--date-plain-p   |         1267 | 0.6198571040 | 0.0004892321 |
| org-entry-get                 |         3983 | 0.2979337370 | 7.480...e-05 |
| org-is-habit-p                |         1365 | 0.2049101109 | 0.0001501172 |
| org--property-local-values    |         1365 | 0.1940614150 | 0.0001421695 |
| org-agenda-ng--habit-p        |         1272 | 0.1911009179 | 0.0001502365 |
| org-agenda-ng--format-element |           52 |  0.177965411 | 0.0034224117 |
| org-get-property-block        |         1365 | 0.1760004519 | 0.0001289380 |
| org-get-tags-at               |           52 | 0.1362824969 | 0.0026208172 |
| org-agenda-ng--date-p         |         3880 | 0.1351176629 | 3.482...e-05 |
| org-up-heading-safe           |          226 | 0.1276747609 | 0.0005649325 |
| re-search-backward            |         2028 | 0.1144211070 | 5.642...e-05 |
| org-entry-properties          |         2618 | 0.0848660999 | 3.241...e-05 |
| org-agenda-ng--todo-p         |         1319 |  0.081952653 | 6.213...e-05 |
| org-get-todo-state            |         1319 | 0.0796836810 | 6.041...e-05 |
| re-search-forward             |         3754 | 0.0739803739 | 1.970...e-05 |
| org-inlinetask-in-task-p      |         1365 | 0.0657829330 | 4.819...e-05 |
| org-agenda-ng--scheduled-p    |         1247 | 0.0619497850 | 4.967...e-05 |

** Intersecting query results

An idea that /might/ be helpful for performance in /some/ cases, depending on the query, the data, and whether the query has a preamble.  But it looks like it would very rarely be helpful.

#+BEGIN_SRC elisp
  (cl-defun org-ql-agenda-intersection (buffers-files queries &key entries sort buffer narrow super-groups)
    "Like `org-ql-agenda', but intersects multiple queries."
    (declare (indent defun))
    (let* ((org-ql-cache (ht))
           (entries (->> queries
                         (--map (org-ql-select buffers-files
                                  it
                                  :action 'element-with-markers
                                  :narrow narrow
                                  :sort sort))
                         (-reduce #'-intersection))))
      (org-ql-agenda--agenda buffers-files queries
        :entries entries :super-groups super-groups)))

  (bench-multi-lexical :times 1
    :forms (("intersection" (let ((org-use-tag-inheritance nil))
                              (org-ql-agenda-intersection (org-agenda-files)
                                '((todo "TODO")
                                  (tags "Emacs"))
                                :sort '(priority deadline)
                                :super-groups org-super-agenda-groups)))
            ("normal" (let ((org-use-tag-inheritance nil))
                        (org-ql-agenda (org-agenda-files)
                          (and (todo "TODO")
                               (tags "Emacs"))
                          :sort (priority deadline)
                          :super-groups org-super-agenda-groups)))))
#+END_SRC

#+RESULTS:
| Form         | x faster than next | Total runtime | # of GCs | Total GC runtime |
|--------------+--------------------+---------------+----------+------------------|
| normal       | 3.70               |      0.233147 |        0 |                0 |
| intersection | slowest            |      0.862512 |        0 |                0 |

*** Alternative approach

[2019-09-01 Sun 08:17]  This is very experimental, but the results are surprising.  When the action function returns a fairly simple list, the intersection is very slightly faster.  When returning full elements, the intersection is much slower, so that it more than doubles the runtime.  I wonder if the element list comparison is short-circuiting, or if it looks at the whole lists, because it seems like it shouldn't take more than 4-5 list elements before it realizes that two lists don't match.

Anyway, looks like this approach isn't viable, at least not without a much more complicated implementation, which probably wouldn't be worth it.

#+BEGIN_SRC elisp
  (let* ((action-fn (lambda ()
                      (list (current-buffer)
                            (point)
                            (substring-no-properties (org-get-heading t t)))))
         (files '("~/org/main.org")))
    ;; NOTE: Careful to use the same files and action in each one.  I duplicated
    ;; the variable in each form to make individual testing easier.
    (bench-multi-lexical :times 1 :ensure-equal t
      :forms (("normal" (->> (let ((org-ql-cache (ht))
                                   (action-fn (lambda ()
                                                (list (current-buffer)
                                                      (point)
                                                      (substring-no-properties (org-get-heading t t)))))
                                   (files '("~/org/main.org")))
                               (org-ql-select files
                                 '(and (not (done))
                                       (or (habit)
                                           (deadline auto)
                                           (scheduled :to today)
                                           (ts-active :on today)
                                           (closed :on today)))
                                 :action action-fn))))
              ("Testing" (let* ((org-ql-cache (ht))
                                (files '("~/org/main.org"))
                                (action-fn (lambda ()
                        (list (current-buffer)
                              (point)
                              (substring-no-properties (org-get-heading t t)))))
                                (and-queries '(not (done)))
                                (or-queries '((habit)
                                              (deadline auto)
                                              (scheduled :to today)
                                              (ts-active :on today)
                                              (closed :on today)))
                                (and-results (org-ql-select files
                                               and-queries
                                               :action action-fn))
                                (or-results (cl-loop for query in or-queries
                                                     append (org-ql-select files
                                                              query
                                                              :action action-fn))))
                           (seq-intersection and-results
                                             (->> or-results
                                                  -uniq)))))))
#+END_SRC

#+RESULTS:
| Form    | x faster than next | Total runtime | # of GCs | Total GC runtime |
|---------+--------------------+---------------+----------+------------------|
| Testing | 1.15               |      0.248376 |        0 |                0 |
| normal  | slowest            |      0.284897 |        0 |                0 |

#+BEGIN_SRC elisp

;; With caching enabled
  (let* ((action-fn (lambda ()
                      (list (current-buffer)
                            (point)
                            (substring-no-properties (org-get-heading t t)))))
         (files '("~/org/main.org")))
    (bench-multi-lexical :times 1 :ensure-equal t
      :forms (("normal" (->> (org-ql-select files
                               '(and (not (done))
                                     (or (habit)
                                         (deadline auto)
                                         (scheduled :to today)
                                         (ts-active :on today)
                                         (closed :on today)))
                               :action action-fn)))
              ("Testing" (let* ((files '("~/org/main.org"))
                                (and-queries '(not (done)))
                                (or-queries '((habit)
                                              (deadline auto)
                                              (scheduled :to today)
                                              (ts-active :on today)
                                              (closed :on today)))
                                (and-results (org-ql-select files
                                               and-queries
                                               :action action-fn))
                                (or-results (cl-loop for query in or-queries
                                                     append (org-ql-select files
                                                              query
                                                              :action action-fn))))
                           (seq-intersection and-results
                                             (->> or-results
                                                  -uniq)))))))
#+END_SRC

#+RESULTS:
| Form    | x faster than next | Total runtime | # of GCs | Total GC runtime |
|---------+--------------------+---------------+----------+------------------|
| normal  | 13.72              |      0.002311 |        0 |                0 |
| Testing | slowest            |      0.031707 |        0 |                0 |

Using full views:

#+BEGIN_SRC elisp
  (let* ((action-fn (lambda ()
                      (list (current-buffer)
                            (point)
                            (substring-no-properties (org-get-heading t t)))))
         (files '("~/org/main.org")))
    (bench-multi-lexical :times 1
      :forms (("normal" (->> (let ((org-ql-cache (ht))
                                   (files '("~/org/main.org")))
                               (org-ql-search files
                                 '(and (not (done))
                                       (or (habit)
                                           (deadline auto)
                                           (scheduled :to today)
                                           (ts-active :on today)
                                           (closed :on today)))))))
              ("Testing" (let* ((org-ql-cache (ht))
                                (files '("~/org/main.org"))
                                (and-queries '(not (done)))
                                (or-queries '((habit)
                                              (deadline auto)
                                              (scheduled :to today)
                                              (ts-active :on today)
                                              (closed :on today)))
                                (and-results (org-ql-select files
                                               and-queries
                                               :action 'element-with-markers))
                                (or-results (cl-loop for query in or-queries
                                                     append (org-ql-select files
                                                              query
                                                              :action 'element-with-markers)))
                                (final-results (seq-intersection and-results
                                                                 (->> or-results
                                                                      -uniq))))
                           (org-ql-agenda--agenda nil nil
                             :entries final-results)
                         
                           )))))
#+END_SRC

#+RESULTS:
| Form    | x faster than next | Total runtime | # of GCs | Total GC runtime |
|---------+--------------------+---------------+----------+------------------|
| normal  | 1.74               |      0.534742 |        0 |                0 |
| Testing | slowest            |      0.931897 |        0 |                0 |

Just gathering results, but using elements:

#+BEGIN_SRC elisp
  (let* ((action-fn 'element-with-markers)
         (files '("~/org/main.org")))
    ;; NOTE: Careful to use the same files and action in each one.  I duplicated
    ;; the variable in each form to make individual testing easier.
    (bench-multi-lexical :times 1 :ensure-equal t
      :forms (("normal" (->> (let ((org-ql-cache (ht)))
                               (org-ql-select files
                                 '(and (not (done))
                                       (or (habit)
                                           (deadline auto)
                                           (scheduled :to today)
                                           (ts-active :on today)
                                           (closed :on today)))
                                 :action action-fn))))
              ("Testing" (let* ((org-ql-cache (ht))
                                (and-queries '(not (done)))
                                (or-queries '((habit)
                                              (deadline auto)
                                              (scheduled :to today)
                                              (ts-active :on today)
                                              (closed :on today)))
                                (and-results (org-ql-select files
                                               and-queries
                                               :action action-fn))
                                (or-results (cl-loop for query in or-queries
                                                     append (org-ql-select files
                                                              query
                                                              :action action-fn))))
                           (seq-intersection and-results
                                             (->> or-results
                                                  -uniq)))))))
#+END_SRC

#+RESULTS:
| Form    | x faster than next | Total runtime | # of GCs | Total GC runtime |
|---------+--------------------+---------------+----------+------------------|
| normal  | 2.27               |      0.314218 |        0 |                0 |
| Testing | slowest            |      0.714587 |        0 |                0 |


** [2019-08-29 Thu 06:24]  Benchmarking org-ql compared to re-search-forward for getting headings in buffer
:PROPERTIES:
:ID:       fcc09229-ed24-42eb-a1fd-31d8f7d4c8d5
:END:

Minimal difference, and that's a very large file, too.  On smaller files it's thousandths of a second.

#+BEGIN_SRC elisp
  (with-current-buffer (find-buffer-visiting "~/org/inbox.org")
    (bench-multi-lexical :times 1 :ensure-equal t
      :forms (("org-ql" (org-ql-select (current-buffer)
                          '(level 1)
                          :action '(progn
                                     (font-lock-ensure (line-beginning-position) (line-end-position))
                                     (cons (org-get-heading t) (point)))))
              ("re-search-forward" (org-with-wide-buffer
                                    (goto-char (point-min))
                                    (when (org-before-first-heading-p)
                                      (outline-next-heading))
                                    (cl-loop while (re-search-forward (rx bol "*" (1+ blank)) nil t)
                                             do (font-lock-ensure (line-beginning-position) (line-end-position))
                                             collect (cons (org-get-heading t) (match-beginning 0))))))))
#+END_SRC

#+RESULTS:
| Form              | x faster than next | Total runtime | # of GCs | Total GC runtime |
|-------------------+--------------------+---------------+----------+------------------|
| re-search-forward | 1.17               |      0.520375 |        0 |                0 |
| org-ql            | slowest            |      0.608281 |        0 |                0 |

** Caching of inherited tags

[2019-09-05 Thu 07:59]  Implemented a per-buffer tags cache that seems to significantly speed up tags queries that use tag inheritance.  It persists as long as the buffer remains unmodified, and it's usable from any code as a single function that automatically uses caching.  It also returns inherited tags and local tags separately, which could be useful for having separate selectors, one for inherited tags, one for local tags, and one for both.

#+BEGIN_SRC elisp
  (defvar org-ql-tags-cache (ht)
    "Per-buffer tags cache.
  Keyed by buffer.  Each value is a cons of the buffer's modified
  tick, and another hash table keyed on buffer position, whose
  values are a list of two lists, inherited tags and local tags, as
  strings.")

  (defun org-ql--tags-at (position)
    "Return tags for POSITION in current buffer.
  Returns cons (INHERITED-TAGS . LOCAL-TAGS)."
    ;; I'd like to use `-if-let*', but it doesn't leave non-nil variables
    ;; bound in the else clause, so destructured variables that are non-nil,
    ;; like found caches, are not available in the else clause.
    (if-let* ((buffer-cache (gethash (current-buffer) org-ql-tags-cache))
              (modified-tick (car buffer-cache))
              (tags-cache (cdr buffer-cache))
              (buffer-unmodified-p (eq (buffer-modified-tick) modified-tick))
              (cached-result (gethash position tags-cache)))
        ;; Found in cache: return them.
        (pcase cached-result
          ('org-ql-nil nil)
          (_ cached-result))
      ;; Not found in cache: get tags and cache them.
      (let* ((local-tags (or (org-get-tags position 'local)
                             'org-ql-nil))
             (inherited-tags (or (save-excursion
                                   (when (org-up-heading-safe)
                                     (-let* (((inherited local) (org-ql--tags-at (point)))
                                             (inherited-tags (when (or inherited local)
                                                               (cond ((and (listp inherited)
                                                                           (listp local))
                                                                      (append inherited local))
                                                                     ((cond ((listp inherited) inherited)
                                                                            ((listp local) local)))))))
                                       (when inherited-tags
                                         (->> inherited-tags -non-nil -uniq)))))
                                 'org-ql-nil))
             (all-tags (list inherited-tags local-tags)))
        ;; Check caches again, because they may have been set now.
        ;; TODO: Is there a clever way we could avoid doing this, or is it inherently necessary?
        (setf buffer-cache (gethash (current-buffer) org-ql-tags-cache)
              modified-tick (car buffer-cache)
              tags-cache (cdr buffer-cache)
              buffer-unmodified-p (eq (buffer-modified-tick) modified-tick))
        (cond ((or (not buffer-cache)
                   (not buffer-unmodified-p))
               ;; Buffer-local tags cache empty or invalid: make new one.
               (puthash (current-buffer)
                        (cons (buffer-modified-tick)
                              (let ((table (make-hash-table)))
                                (puthash position all-tags table)
                                table))
                        org-ql-tags-cache)
               ;; Return tags, not the cons put on the buffer-cache.
               all-tags)
              ;; Buffer-local tags cache found, but no result: store this one.
              (t (puthash position all-tags tags-cache))))))

  (org-ql--defpred tags-cached (&rest tags)
    "Return non-nil if current heading has one or more of TAGS (a list of strings)."
    ;; TODO: Try to use `org-make-tags-matcher' to improve performance.  It would be nice to not have
    ;; to run `org-get-tags' for every heading, especially with inheritance.
    (cl-macrolet ((tags-p (tags)
                          `(and ,tags
                                (not (eq 'org-ql-nil ,tags)))))
      (-let* (((inherited local) (org-ql--tags-at (point))))
        (cl-typecase tags
          (null (or (tags-p inherited)
                    (tags-p local)))
          (otherwise (or (when (tags-p inherited)
                           (seq-intersection tags inherited))
                         (when (tags-p local)
                           (seq-intersection tags local))))))))
#+END_SRC

Benchmark results:

#+BEGIN_SRC elisp
  (let* ((buffers '("~/org/main.org"))
         (tags '("Emacs")))
    (bench-multi-lexical :times 1 :ensure-equal t
      :forms (("uncached" (let ((org-ql-cache (ht)))
                            (org-ql-select buffers
                              `(tags ,@tags))))
              ("cached" (let ((org-ql-cache (ht))
                              (org-ql-tags-cache (ht)))
                          (org-ql-select buffers
                            `(tags-cached ,@tags)))))))
#+END_SRC

#+RESULTS:
| Form     | x faster than next | Total runtime | # of GCs | Total GC runtime |
|----------+--------------------+---------------+----------+------------------|
| cached   | 6.51               |      0.519871 |        0 |                0 |
| uncached | slowest            |      3.386679 |        0 |                0 |


