Fixes https://github.com/python-caldav/caldav/issues/156

* Adds new function in lib/vcal.py for generating icalendar data
  (arguably that one may fit better to the icalendar module?)

* The calendar methods save_event, save_todo and save_journal
  now accepts any number of extra parameter, and it will be
  added to the icalendar data.

* Some test code, documentation and example code

7 files changed, 209 insertions, 64 deletions

diff --git a/caldav/lib/vcal.py b/caldav/lib/vcal.py
index f0e2cf4..2b132ae 100644
--- a/caldav/lib/vcal.py
+++ b/caldav/lib/vcal.py
@@ -3,7 +3,9 @@
 
 
 import re
-from caldav.lib.python_utilities import to_local
+from caldav.lib.python_utilities import to_local, to_wire
+import datetime
+import uuid
 
 ## Fixups to the icalendar data to work around compatbility issues.
 
@@ -75,3 +77,37 @@ def fix(event):
             fixed2 += line + "\n"
 
     return fixed2
+
+## sorry for being english-language-euro-centric ... fits rather perfectly as default language for me :-)
+def create_ical(ical_fragment=None, objtype=None, language='en_DK', **attributes):
+    """
+    I somehow feel this fits more into the icalendar library than here
+    """
+    ## late import, icalendar is not an explicit requirement for v0.x of the caldav library.
+    ## (perhaps I should change my position on that soon)
+    import icalendar
+    if ical_fragment:
+        ical_fragment = to_wire(ical_fragment)
+    if not ical_fragment or not re.search(b'^BEGIN:V', ical_fragment, re.MULTILINE):
+        my_instance = icalendar.Calendar()
+        my_instance.add('prodid', f'-//python-caldav//caldav//{language}')
+        my_instance.add('version', '2.0')
+        if objtype is None:
+            objtype='VEVENT'
+        component = icalendar.cal.component_factory[objtype]()
+        component.add('dtstamp', datetime.datetime.now())
+        component.add('uid', uuid.uuid1())
+        my_instance.add_component(component)
+    else:
+        if not ical_fragment.startswith(b'BEGIN:VCALENDAR'):
+            ical_fragment = b"BEGIN:VCALENDAR\n"+to_wire(ical_fragment.strip())+b"\nEND:VCALENDAR\n"
+        my_instance = icalendar.Calendar.from_ical(ical_fragment)
+        component = my_instance.subcomponents[0]
+        ical_fragment = None
+    for attribute in attributes:
+        if attributes[attribute] is not None:
+            component.add(attribute, attributes[attribute])
+    ret = my_instance.to_ical()
+    if ical_fragment is not None:
+        ret = re.sub(b"^END:V", ical_fragment.strip() + b"\nEND:V", ret, flags=re.MULTILINE)
+    return ret
diff --git a/caldav/objects.py b/caldav/objects.py
index 1730dca..8ebf2fa 100644
--- a/caldav/objects.py
+++ b/caldav/objects.py
@@ -31,7 +31,7 @@ except:
 from caldav.lib import error, vcal
 from caldav.lib.url import URL
 from caldav.elements import dav, cdav, ical
-from caldav.lib.python_utilities import to_unicode
+from caldav.lib.python_utilities import to_unicode, to_wire
 
 import logging
 log = logging.getLogger('caldav')
@@ -593,34 +593,42 @@ class Calendar(DAVObject):
         obj.id = obj.icalendar_instance.walk('vevent')[0]['uid']
         obj.save()
 
-    def save_event(self, ical, no_overwrite=False, no_create=False):
+    def _use_or_create_ics(self, ical, objtype, **ical_data):
+        if ical_data or ((isinstance(ical, str) or isinstance(ical, bytes)) and not b'BEGIN:V' in to_wire(ical)):
+            return vcal.create_ical(ical_fragment=ical, objtype=objtype, **ical_data)
+        return ical
+
+    def save_event(self, ical=None, no_overwrite=False, no_create=False, **ical_data):
         """
         Add a new event to the calendar, with the given ical.
 
         Parameters:
          * ical - ical object (text)
+         * no_overwrite - existing calendar objects should not be overwritten
+         * no_create - don't create a new object, existing calendar objects should be updated
+         * ical_data - passed to lib.vcal.create_ical
         """
-        e = Event(self.client, data=ical, parent=self)
+        e = Event(self.client, data=self._use_or_create_ics(ical, objtype='VEVENT', **ical_data), parent=self)
         e.save(no_overwrite=no_overwrite, no_create=no_create, obj_type='event')
         return e
 
-    def save_todo(self, ical, no_overwrite=False, no_create=False):
+    def save_todo(self, ical=None, no_overwrite=False, no_create=False, **ical_data):
         """
         Add a new task to the calendar, with the given ical.
 
         Parameters:
          * ical - ical object (text)
         """
-        return Todo(self.client, data=ical, parent=self).save(no_overwrite=no_overwrite, no_create=no_create, obj_type='todo')
+        return Todo(self.client, data=self._use_or_create_ics(ical, objtype='VTODO', **ical_data), parent=self).save(no_overwrite=no_overwrite, no_create=no_create, obj_type='todo')
 
-    def save_journal(self, ical, no_overwrite=False, no_create=False):
+    def save_journal(self, ical=None, no_overwrite=False, no_create=False, **ical_data):
         """
         Add a new journal entry to the calendar, with the given ical.
 
         Parameters:
          * ical - ical object (text)
         """
-        return Journal(self.client, data=ical, parent=self).save(no_overwrite=no_overwrite, no_create=no_create, obj_type='journal')
+        return Journal(self.client, data=self._use_or_create_ics(ical, objtype='VJOURNAL', **ical_data), parent=self).save(no_overwrite=no_overwrite, no_create=no_create, obj_type='journal')
 
     ## legacy aliases
     add_event = save_event
diff --git a/docs/source/index.rst b/docs/source/index.rst
index a9e9d41..f0acdb4 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -26,7 +26,7 @@ Objective and scope
 
 The python caldav library should make interactions with calendar servers
 simple and easy.  Simple operations (like find a list of all calendars
-owned, inserting an icalendar object into a calendar, do a simple date
+owned, inserting a new event into a calendar, do a simple date
 search, etc) should be trivial to accomplish even if the end-user of
 the library has no or very little knowledge of the caldav, webdav or
 icalendar standards.  The library should be agile enough to allow
@@ -58,6 +58,8 @@ support RFC 5545 (icalendar).  It's outside the scope of this library
 to implement logic for parsing and modifying RFC 5545, instead we
 depend on another library for that.
 
+RFC 5545 describes the icalendar format.  Constructing or parsing icalendar data was considered out of the scope of this library, but we do make exceptions - like, there is a method to complete a task - it involves editing the icalendar data, and now the `save_event`, `save_todo` and `save_journal` methods are able to construct icalendar data if needed.
+
 There exists two libraries supporting RFC 5545, vobject and icalendar.
 The icalendar library seems to be more popular.  Version 0.x depends
 on vobject, version 1.x will depend on icalendar.  Version 0.7 and
@@ -90,7 +92,9 @@ order is available for "power users".
 Quickstart
 ==========
 
-All code examples below are snippets from the basic_usage_examples.py.
+All code examples below was snippets from the basic_usage_examples.py,
+but the documentation and the examples may have drifted apart (TODO:
+does there exist some good system for this?  Just use docstrings and doctests?)
 
 Setting up a caldav client object and a principal object:
 
@@ -111,7 +115,17 @@ Creating a calendar:
 
   my_new_calendar = my_principal.make_calendar(name="Test calendar")
 
-Adding an event to the calendar:
+Adding an event to the calendar, v0.9 adds this interface:
+
+.. code-block:: python
+
+  my_event = my_new_calendar.save_event(
+      dtstart=datetime.datetime(2020,5,17,8),
+      dtend=datetime.datetime(2020,5,18,1),
+      summary="Do the needful",
+      rrule={'FREQ': 'YEARLY'))
+
+Adding an event described through some ical text:
 
 .. code-block:: python
 
@@ -172,24 +186,17 @@ Create a task list:
   my_new_tasklist = my_principal.make_calendar(
               name="Test tasklist", supported_calendar_component_set=['VTODO'])
 
-Adding a task to a task list:
+Adding a task to a task list.  The ics parameter may be some complete ical text string or a fragment.
 
 .. code-block:: python
 
-  my_new_tasklist.add_todo("""BEGIN:VCALENDAR
-  VERSION:2.0
-  PRODID:-//Example Corp.//CalDAV Client//EN
-  BEGIN:VTODO
-  UID:20070313T123432Z-456553@example.com
-  DTSTAMP:20070313T123432Z
-  DTSTART;VALUE=DATE:20200401
-  DUE;VALUE=DATE:20200501
-  RRULE:FREQ=YEARLY
-  SUMMARY:Deliver some data to the Tax authorities
-  CATEGORIES:FAMILY,FINANCE
-  STATUS:NEEDS-ACTION
-  END:VTODO
-  END:VCALENDAR""")
+  my_new_tasklist.save_todo(
+      ics = "RRULE:FREQ=YEARLY",
+      summary="Deliver some data to the Tax authorities",
+      dtstart=date(2020, 4, 1),
+      due=date(2020,5,1),
+      categories=['family', 'finance'],
+      status='NEEDS-ACTION')
 
 Fetching tasks:
 
diff --git a/examples/basic_usage_examples.py b/examples/basic_usage_examples.py
index baae0cf..1ab3c2b 100644
--- a/examples/basic_usage_examples.py
+++ b/examples/basic_usage_examples.py
@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, date
 import sys
 
 ## We'll try to use the local caldav library, not the system-installed
@@ -57,25 +57,26 @@ except caldav.error.NotFoundError:
     my_new_calendar = my_principal.make_calendar(name="Test calendar")
 
 ## Let's add an event to our newly created calendar
-my_event = my_new_calendar.save_event("""BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Example Corp.//CalDAV Client//EN
-BEGIN:VEVENT
-UID:20200516T060000Z-123401@example.com
-DTSTAMP:20200516T060000Z
-DTSTART:20200517T060000Z
-DTEND:20200517T230000Z
-RRULE:FREQ=YEARLY
-SUMMARY:Do the needful
-END:VEVENT
-END:VCALENDAR
-""")
+## (This usage pattern is new from v0.9.
+## Earlier save_event would only accept some ical data)
+my_event = my_new_calendar.save_event(
+    dtstart=datetime(2020,5,17,8),
+    dtend=datetime(2020,5,18,1),
+    summary="Do the needful",
+    rrule={'FREQ': 'YEARLY'})
 
 ## Let's search for the newly added event.
+## (this may fail if the server doesn't support expand)
 print("Here is some icalendar data:")
-events_fetched = my_new_calendar.date_search(
-    start=datetime(2021, 1, 1), end=datetime(2024, 1, 1), expand=True)
-print(events_fetched[0].data)
+try:
+    events_fetched = my_new_calendar.date_search(
+        start=datetime(2021, 5, 16), end=datetime(2024, 1, 1), expand=True)
+    print(events_fetched[0].data)
+except:
+    print("Your calendar server does apparently not support expanded search")
+    events_fetched = my_new_calendar.date_search(
+        start=datetime(2020, 5, 16), end=datetime(2024, 1, 1), expand=False)
+    print(events_fetched[0].data)
 
 event = events_fetched[0]
 
@@ -83,7 +84,8 @@ event = events_fetched[0]
 ## The caldav library has always been supporting vobject out of the box, but icalendar is more popular.
 ## event.instance will as of version 0.x yield a vobject instance, but this may change in future versions.
 ## Both event.vobject_instance and event.icalendar_instance works from 0.7.
-event.vobject_instance.vevent.summary.value = 'Norwegian national day celebrations'
+event.vobject_instance.vevent.summary.value = 'Norwegian national day celebratiuns'
+event.icalendar_instance.subcomponents[0]['summary'] = event.icalendar_instance.subcomponents[0]['summary'].replace('celebratiuns', 'celebrations')
 event.save()
 
 ## Please note that the proper way to save new icalendar data
@@ -110,6 +112,7 @@ assert(len(all_events) == len(list(all_objects)))
 
 ## Let's check that the summary got right
 assert all_events[0].vobject_instance.vevent.summary.value.startswith('Norwegian')
+assert all_events[0].vobject_instance.vevent.summary.value.endswith('celebrations')
 
 ## This calendar should as a minimum support VEVENTs ... most likely
 ## it also supports VTODOs and maybe even VJOURNALs.  We can query the
@@ -125,24 +128,18 @@ my_new_tasklist = my_principal.make_calendar(
             name="Test tasklist", supported_calendar_component_set=['VTODO'])
 
 ## We'll add a task to the task list
-my_new_tasklist.add_todo("""BEGIN:VCALENDAR
-VERSION:2.0
-PRODID:-//Example Corp.//CalDAV Client//EN
-BEGIN:VTODO
-UID:20070313T123432Z-456553@example.com
-DTSTAMP:20070313T123432Z
-DTSTART;VALUE=DATE:20200401
-DUE;VALUE=DATE:20200501
-RRULE:FREQ=YEARLY
-SUMMARY:Deliver some data to the Tax authorities
-CATEGORIES:FAMILY,FINANCE
-STATUS:NEEDS-ACTION
-END:VTODO
-END:VCALENDAR""")
+my_new_tasklist.add_todo(
+    ics = "RRULE:FREQ=YEARLY",
+    summary="Deliver some data to the Tax authorities",
+    dtstart=date(2020, 4, 1),
+    due=date(2020,5,1),
+    categories=['family', 'finance'],
+    status='NEEDS-ACTION')
 
 ## Fetch the tasks
 todos = my_new_tasklist.todos()
 assert(len(todos) == 1)
+assert('FREQ=YEARLY' in todos[0].data)
 
 print("Here is some more icalendar data:")
 print(todos[0].data)
diff --git a/setup.py b/setup.py
index d16e46c..7e0d385 100755
--- a/setup.py
+++ b/setup.py
@@ -5,15 +5,17 @@ import sys
 
 ## ATTENTION! when doing releases, the default debugmode in lib/error.py should be set to PRODUCTION.
 ## (TODO: any nicer ways than doing this manually?  Make a "releases" branch, maybe?)
-version = '0.8.1'
+version = '0.9.0dev'
 
 if __name__ == '__main__':
-    ## For python 2.7 and 3.5 we depend on pytz and tzlocal.  For 3.6 and up, batteries are included.  Same with mock.
+    ## For python 2.7 and 3.5 we depend on pytz and tzlocal.  For 3.6 and up, batteries are included.  Same with mock. (But unfortunately the icalendar library only support pytz timezones, so we'll keep pytz around for a bit longer).
     try:
         import datetime
         from datetime import timezone
-        datetime.datetime.now().astimezone()
+        datetime.datetime.now().astimezone(timezone.utc)
         extra_packages = []
+        ## line below can be removed when https://github.com/collective/icalendar/issues/333 is fixed
+        extra_packages = ['pytz', 'tzlocal']
     except:
         extra_packages = ['pytz', 'tzlocal']
     try:
diff --git a/tests/test_caldav.py b/tests/test_caldav.py
index 935478e..7edd5b8 100644
--- a/tests/test_caldav.py
+++ b/tests/test_caldav.py
@@ -18,7 +18,7 @@ import uuid
 import tempfile
 import random
 from collections import namedtuple
-from datetime import datetime
+from datetime import datetime, date
 from six import PY3
 from nose.tools import assert_equal, assert_not_equal, assert_raises
 from nose.plugins.skip import SkipTest
@@ -685,6 +685,11 @@ class RepeatedFunctionalTestsBaseClass(object):
             events2 = c2.events()
             assert_equal(len(events2), 1)
             assert_equal(events2[0].url, events[0].url)
+            
+        # add another event, it should be doable without having premade ICS
+        ev2 = c.save_event(dtstart=datetime(2015, 10, 10, 8, 7, 6), summary="This is a test event", dtend=datetime(2016, 10, 10, 9, 8, 7))
+        events = c.events()
+        assert_equal(len(events), len(existing_events) + 2)
 
     def testCalendarByFullURL(self):
         """
@@ -991,6 +996,8 @@ class RepeatedFunctionalTestsBaseClass(object):
         assert_equal(len(journals), 1)
         j1_ = c.journal_by_uid(j1.id)
         assert_equal(j1_.data, journals[0].data)
+        j2 = c.save_journal(dtstart=date(2011,11,11), summary="A childbirth in a hospital in Kupchino", description="A quick birth, in the middle of the night")
+        assert_equal(len(c.journals()), 2)
         todos = c.todos()
         events = c.events()
         assert_equal(todos + events, [])
@@ -1033,11 +1040,15 @@ class RepeatedFunctionalTestsBaseClass(object):
         assert_equal(len(todos), 1)
         assert_equal(len(todos2), 1)
 
+        t3 = c.save_todo(summary="mop the floor", categories=["housework"], priority=4)
+        assert_equal(len(c.todos()), 2)
+        
         # adding a todo without an UID, it should also work
         if not self.check_compatibility_flag('uid_required'):
             c.save_todo(todo7)
-            assert_equal(len(c.todos()), 2)
-
+            assert_equal(len(c.todos()), 3)
+        
+            
         logging.info("Fetching the events (should be none)")
         # c.events() should NOT return todo-items
         events = c.events()
diff --git a/tests/test_vcal.py b/tests/test_vcal.py
new file mode 100644
index 0000000..42847b9
--- /dev/null
+++ b/tests/test_vcal.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+from unittest import TestCase
+import vobject
+import icalendar
+import uuid
+from caldav.lib.vcal import fix, create_ical
+#from datetime import timezone
+import pytz
+from datetime import datetime, timedelta
+from caldav.lib.python_utilities import to_normal_str, to_wire
+
+#utc = timezone.utc
+import pytz
+utc = pytz.utc
+
+# example from http://www.rfc-editor.org/rfc/rfc5545.txt
+ev = """BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Example Corp.//CalDAV Client//EN
+BEGIN:VEVENT
+UID:19970901T130000Z-123403@example.com
+DTSTAMP:19970901T130000Z
+DTSTART;VALUE=DATE:19971102
+SUMMARY:Our Blissful Anniversary
+TRANSP:TRANSPARENT
+CLASS:CONFIDENTIAL
+CATEGORIES:ANNIVERSARY,PERSONAL,SPECIAL OCCASION
+RRULE:FREQ=YEARLY
+END:VEVENT
+END:VCALENDAR"""
+
+
+class TestVcal(TestCase):
+    def assertSameICal(self, ical1, ical2):
+        """helper method"""
+        def normalize(s):
+            s = to_wire(s).replace(b'\r\n',b'\n').strip().split(b'\n')
+            s.sort()
+            return b"\n".join(s)
+        self.assertEqual(normalize(ical1), normalize(ical2))
+        return ical2
+
+    def verifyICal(self, ical):
+        """
+        Does a best effort on verifying that the ical is correct, by
+        pushing it through the vobject and icalendar library
+        """
+        vobj = vobject.readOne(to_normal_str(ical))
+        icalobj = icalendar.Calendar.from_ical(ical)
+        self.assertSameICal(icalobj.to_ical(), ical)
+        self.assertSameICal(vobj.serialize(), ical)
+        return icalobj.to_ical()
+    
+    ## TODO: create a test_fix, should be fairly simple - for each
+    ## "fix" that's done in the code, make up some broken ical data
+    ## that demonstrates the brokenness we're dealing with (preferably
+    ## real-world examples). Then ...
+    #for bical in broken_ical:
+    #    verifyICal(vcal.fix(bical))
+
+    def test_create_ical(self):
+        def create_and_validate(**args):
+            return self.verifyICal(create_ical(**args))
+
+        ## First, a fully valid ical_fragment should go through as is
+        self.assertSameICal(create_and_validate(ical_fragment=ev), ev)
+
+        ## One may add stuff to a fully valid ical_fragment
+        self.assertSameICal(create_and_validate(ical_fragment=ev, priority=3), ev+"\nPRIORITY:3\n")
+
+        ## The returned ical_fragment should always contain BEGIN:VCALENDAR and END:VCALENDAR
+        ical_fragment = ev.replace('BEGIN:VCALENDAR', '').replace('END:VCALENDAR', '')
+        self.assertSameICal(create_and_validate(ical_fragment=ical_fragment), ev)
+        
+        ## Create something with a dtstart and verify that we get it back in the ical
+        some_ical = create_and_validate(summary="gobledok", dtstart=datetime(2032,10,10,10,10,10, tzinfo=utc), duration=timedelta(hours=5))
+        self.assertTrue(b'DTSTART;VALUE=DATE-TIME:20321010T101010Z' in some_ical)
+
+        ## Verify that ical_fragment works as intended
+        some_ical = create_and_validate(summary="gobledok", ical_fragment="PRIORITY:3", dtstart=datetime(2032,10,10,10,10,10, tzinfo=utc), duration=timedelta(hours=5))
+        self.assertTrue(b'DTSTART;VALUE=DATE-TIME:20321010T101010Z' in some_ical)
+        
+