diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba3532d3b6f..84f07bae985 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,8 @@ # Contributing to Dataverse -Thank you for your interest in contributing to Dataverse! We welcome contributions of ideas, bug reports, usability testing, documentation, code, and more! +Thank you for your interest in contributing to Dataverse! We are open to contributions from everyone. You don't need permission to participate, just jump in using the resources below. If you have questions, reach out to us on the [#dataverse IRC channel][], and hang around a while, as it may take time for community members to de-idle. + +We aren't just looking for developers, there are many ways to contribute to Dataverse. We welcome contributions of ideas, bug reports, usability research/feedback, documentation, code, and more! ## Ideas/Feature Requests @@ -26,7 +28,7 @@ An issue is a bug (a feature is no longer behaving the way it should) or a featu Before submitting an issue, please search the existing issues by using the search bar at the top of the page. If there is an existing issue that matches the issue you want to report, please add a comment to it. -If there is no pre-existing issue, please click on the "New Issue" button, log in, and write in what the issue is (unless it is a security issue which should be reported privately to security@dataverse.org). Someone on the Dataverse development team will appropriately tag and assign it to a member of the Dataverse development team. +If there is no pre-existing issue, please click on the "New Issue" button, log in, and write in what the issue is (unless it is a security issue which should be reported privately to security@dataverse.org). If you do not receive a reply to your new issue or comment in a timely manner, please email support@dataverse.org with a link to the issue. @@ -51,12 +53,11 @@ The source for the documentation at http://guides.dataverse.org is in the GitHub Before you start coding, please reach out to us either on the [dataverse-community Google Group][], the [dataverse-dev Google Group][], [IRC][] (#dataverse on freenode), or via support@dataverse.org to make sure the effort is well coordinated and we avoid merge conflicts. -We will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. - -We hope you find the Developer Guide at http://guides.dataverse.org/en/latest/developers helpful. +Please read http://guides.dataverse.org/en/latest/developers/version-control.html to understand how we use the "git flow" model of development and how we will encourage you to create a GitHub issue (if it doesn't exist already) to associate with your pull request. -After making your pull request, your goal should be to help it advance through our kanban board at https://waffle.io/IQSS/dataverse . If no one has moved your pull request to the code review column in a timely manner, please reach out. Thanks! +After making your pull request, your goal should be to help it advance through our kanban board at https://waffle.io/IQSS/dataverse . If no one has moved your pull request to the code review column in a timely manner, please reach out. We maintain a list of [community contributors][] so please let us know if you'd like to be added or removed from the list. Thanks! [dataverse-community Google Group]: https://groups.google.com/group/dataverse-community [dataverse-dev Google Group]: https://groups.google.com/group/dataverse-dev [IRC]: http://chat.dataverse.org +[community contributors]: https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 91a2fc0d5f5..37f2ef7bd68 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -4,7 +4,7 @@ Welcome! New contributors should at least glance at [CONTRIBUTING.md](/CONTRIBUT ## Related Issues -- connects to [#issue number]: [issue title] +- connects to #ISSUE_NUMBER: ISSUE_TITLE ## Pull Request Checklist diff --git a/README.md b/README.md index 0bcc0132b33..93f2f8f754f 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,36 @@ Dataverse® =============== -Dataverse is an open source web application for sharing, citing, analyzing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/)). +Dataverse is an [open source][] web application for sharing, citing, analyzing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]). -Institutions and organizations can choose to install the Dataverse software for their own use. -In this case, the institution will be responsible for maintaining the application; installing upgrades, -setting up backups and data replication as needed. Dataverse 4.0 is now released and used by the Harvard Dataverse: [dataverse.harvard.edu](http://dataverse.harvard.edu/). If you'd like to first test it, you can use our demo site: [demo.dataverse.org](http://demo.dataverse.org). +[dataverse.org][] is our home on the web and shows a map of Dataverse installations around the world, a list of [features][], [integrations][] that have been made possible through [REST APIs][], our development [roadmap][], and more. -For more general information about Dataverse please visit [dataverse.org](http://dataverse.org). +We maintain a demo site at [demo.dataverse.org][] which you are welcome to use for testing and evaluating Dataverse. -The Dataverse code is *open-source* and *free*. +To install Dataverse, please see our [Installation Guide][] which will prompt you to download our [latest release][]. -Dataverse releases are available for download from https://github.com/IQSS/dataverse/releases and installation instructions can be found in the [Installation Guide](http://guides.dataverse.org/en/latest/installation/). +To discuss Dataverse with the community, please join our [mailing list][], participate in a [community call][], chat with us at [chat.dataverse.org][], or attend our annual [Dataverse Community Meeting][]. + +We love contributors! Please see our [Contributing Guide][] for ways you can help. Dataverse is a trademark of President and Fellows of Harvard College and is registered in the United States. [![Dataverse Project logo](src/main/webapp/resources/images/dataverseproject_logo.jpg?raw=true "Dataverse Project")](http://dataverse.org) -[![Build Status](https://travis-ci.org/IQSS/dataverse.svg?branch=master)](https://travis-ci.org/IQSS/dataverse) [![Coverage Status](https://coveralls.io/repos/IQSS/dataverse/badge.svg?branch=master&service=github)](https://coveralls.io/github/IQSS/dataverse?branch=master) +[![Build Status](https://travis-ci.org/IQSS/dataverse.svg?branch=develop)](https://travis-ci.org/IQSS/dataverse) [![Coverage Status](https://coveralls.io/repos/IQSS/dataverse/badge.svg?branch=develop&service=github)](https://coveralls.io/github/IQSS/dataverse?branch=develop) + +[dataverse.org]: https://dataverse.org +[demo.dataverse.org]: https://demo.dataverse.org +[Dataverse community]: https://dataverse.org/developers +[Installation Guide]: http://guides.dataverse.org/en/latest/installation/index.html +[latest release]: https://github.com/IQSS/dataverse/releases +[features]: https://dataverse.org/software-features +[roadmap]: https://dataverse.org/goals-roadmap-and-releases +[integrations]: https://dataverse.org/integrations +[REST APIs]: http://guides.dataverse.org/en/latest/api/index.html +[Contributing Guide]: CONTRIBUTING.md +[mailing list]: https://groups.google.com/group/dataverse-community +[community call]: https://dataverse.org/community-calls +[chat.dataverse.org]: http://chat.dataverse.org +[Dataverse Community Meeting]: https://dataverse.org/events +[open source]: LICENSE.md diff --git a/doc/sphinx-guides/source/_static/api/user-add.json b/doc/sphinx-guides/source/_static/api/user-add.json new file mode 100644 index 00000000000..1be7804bbcf --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/user-add.json @@ -0,0 +1,8 @@ +{ + "firstName": "Lisa", + "lastName": "Simpson", + "userName": "lsimpson", + "affiliation": "Springfield", + "position": "Student", + "email": "lsimpson@mailinator.com" +} diff --git a/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-footer.html b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-footer.html new file mode 100644 index 00000000000..372dcbfd2f2 --- /dev/null +++ b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-footer.html @@ -0,0 +1,38 @@ + + diff --git a/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-header.html b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-header.html new file mode 100644 index 00000000000..454b5e94606 --- /dev/null +++ b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-header.html @@ -0,0 +1,35 @@ + +
+
+

+ SAMPLE IMAGE + SAMPLE HEADER LINK +

+
+
diff --git a/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-homepage.html b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-homepage.html new file mode 100644 index 00000000000..cdaad0ba05c --- /dev/null +++ b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-homepage.html @@ -0,0 +1,13 @@ + + + +
+

Hello, world!

+

Welcome to our Dataverse.

+

Browse Data

+
diff --git a/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-stylesheet.css b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-stylesheet.css new file mode 100644 index 00000000000..e7e284ab51e --- /dev/null +++ b/doc/sphinx-guides/source/_static/installation/files/var/www/dataverse/branding/custom-stylesheet.css @@ -0,0 +1,6 @@ +/* Add any CSS necessary for your custom content */ + +.navbar-default { + background-color: gold; + background-image: none; +} diff --git a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst index b12f0b26fe8..0af163a2916 100644 --- a/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst +++ b/doc/sphinx-guides/source/admin/geoconnect-worldmap.rst @@ -1,10 +1,11 @@ Geoconnect and WorldMap ======================= -.. contents:: :local: - One of the optional components listed under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide is `Geoconnect `_, a piece of middleware that allows Dataverse users to create maps in `WorldMap `_ based on geospatial data stored in Dataverse. For more details on the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. +.. contents:: |toctitle| + :local: + Update "mapitlink" ------------------ diff --git a/doc/sphinx-guides/source/admin/harvestclients.rst b/doc/sphinx-guides/source/admin/harvestclients.rst index 3b7be95f790..25c3d493a38 100644 --- a/doc/sphinx-guides/source/admin/harvestclients.rst +++ b/doc/sphinx-guides/source/admin/harvestclients.rst @@ -1,8 +1,9 @@ Managing Harvesting Clients =========================== -.. contents:: :local: - +.. contents:: |toctitle| + :local: + Your Dataverse as a Metadata Harvester -------------------------------------- diff --git a/doc/sphinx-guides/source/admin/harvestserver.rst b/doc/sphinx-guides/source/admin/harvestserver.rst index 333ae27e925..122ceb0ee28 100644 --- a/doc/sphinx-guides/source/admin/harvestserver.rst +++ b/doc/sphinx-guides/source/admin/harvestserver.rst @@ -1,7 +1,8 @@ Managing Harvesting Server and Sets =================================== -.. contents:: :local: +.. contents:: |toctitle| + :local: Your Dataverse as an OAI server ------------------------------- diff --git a/doc/sphinx-guides/source/admin/index.rst b/doc/sphinx-guides/source/admin/index.rst index 5224e9e2a19..e632192aed8 100755 --- a/doc/sphinx-guides/source/admin/index.rst +++ b/doc/sphinx-guides/source/admin/index.rst @@ -10,7 +10,7 @@ This guide documents the functionality only available to the Dataverse Admin ("N These "superuser" tasks are managed via the new page called the Dashboard. A user logged in as a Dataverse Admin will see the Dashboard link rendered in the upper right corner of every Dataverse page. -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/admin/metadataexport.rst b/doc/sphinx-guides/source/admin/metadataexport.rst index a8931b87716..8c50ceacd84 100644 --- a/doc/sphinx-guides/source/admin/metadataexport.rst +++ b/doc/sphinx-guides/source/admin/metadataexport.rst @@ -1,7 +1,8 @@ Metadata Export =============== -.. contents:: :local: +.. contents:: |toctitle| + :local: Automatic Exports ----------------- diff --git a/doc/sphinx-guides/source/admin/timers.rst b/doc/sphinx-guides/source/admin/timers.rst index 1a5c8874661..f118604654b 100644 --- a/doc/sphinx-guides/source/admin/timers.rst +++ b/doc/sphinx-guides/source/admin/timers.rst @@ -3,10 +3,10 @@ Dataverse Application Timers ============================ -.. contents:: :local: - Dataverse uses timers to automatically run scheduled Harvest and Metadata export jobs. +.. contents:: |toctitle| + :local: Dedicated timer server in a Dataverse server cluster ---------------------------------------------------- diff --git a/doc/sphinx-guides/source/admin/troubleshooting.rst b/doc/sphinx-guides/source/admin/troubleshooting.rst index 792d1067c56..fe9e8b7c659 100644 --- a/doc/sphinx-guides/source/admin/troubleshooting.rst +++ b/doc/sphinx-guides/source/admin/troubleshooting.rst @@ -3,10 +3,11 @@ Troubleshooting =============== -.. contents:: :local: - This new (as of v.4.6) section of the Admin guide is for tips on how to diagnose and fix system problems. +.. contents:: |toctitle| + :local: + Deployment fails, "EJB Timer Service not available" --------------------------------------------------- diff --git a/doc/sphinx-guides/source/api/apps.rst b/doc/sphinx-guides/source/api/apps.rst index bec5a4c9abf..f7c6c75ce0a 100755 --- a/doc/sphinx-guides/source/api/apps.rst +++ b/doc/sphinx-guides/source/api/apps.rst @@ -5,7 +5,8 @@ The introduction of Dataverse APIs has fostered the development of apps that are The apps below are open source, demonstrating how to use Dataverse APIs. Some of these apps (and others) are built on :doc:`/api/client-libraries` that are available for Dataverse APIs. -.. contents:: :local: +.. contents:: |toctitle| + :local: Javascript ---------- diff --git a/doc/sphinx-guides/source/api/client-libraries.rst b/doc/sphinx-guides/source/api/client-libraries.rst index 21a6c384d35..7da51ad03d9 100755 --- a/doc/sphinx-guides/source/api/client-libraries.rst +++ b/doc/sphinx-guides/source/api/client-libraries.rst @@ -5,6 +5,9 @@ Currently there are client libraries for Python, R, and Java that can be used to Because Dataverse is a SWORD server, additional client libraries exist for Java, Ruby, and PHP per the :doc:`/api/sword` page. +.. contents:: |toctitle| + :local: + Python ------ @@ -15,7 +18,7 @@ https://github.com/IQSS/dataverse-client-python is the offical Python package fo R - -https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse APIs. +https://github.com/IQSS/dataverse-client-r is the official R package for Dataverse APIs. The latest release can be installed from `CRAN `_. It was created by `Thomas Leeper `_ whose dataverse can be found at https://dataverse.harvard.edu/dataverse/leeper diff --git a/doc/sphinx-guides/source/api/dataaccess.rst b/doc/sphinx-guides/source/api/dataaccess.rst index 66ea551d446..574cc49616a 100755 --- a/doc/sphinx-guides/source/api/dataaccess.rst +++ b/doc/sphinx-guides/source/api/dataaccess.rst @@ -1,12 +1,11 @@ Data Access API =============== -.. contents:: :local: - - The Data Access API provides programmatic download access to the files stored under Dataverse. More advanced features of the Access API include format-specific transformations (thumbnail generation/resizing for images; converting tabular data into alternative file formats) and access to the data-level metadata that describes the contents of the tabular files. +.. contents:: |toctitle| + :local: Basic File Access ----------------- diff --git a/doc/sphinx-guides/source/api/index.rst b/doc/sphinx-guides/source/api/index.rst index fad6c4c00b1..4fb1f362e14 100755 --- a/doc/sphinx-guides/source/api/index.rst +++ b/doc/sphinx-guides/source/api/index.rst @@ -6,19 +6,11 @@ API Guide ========= -We encourage anyone interested in building tools to -interoperate with the Dataverse to utilize our -APIs. In 4.0, we require to get a token, by simply registering for a Dataverse account, before using our APIs -(We are considering making some of the APIs completely public in the future - no token required - if you use it only a few times). - -Rather than using a production installation of Dataverse, API users are welcome to use http://demo.dataverse.org for testing. - -Please note that the APIs in this guide are shipped with the Dataverse software itself but additional APIs are available if you install the "miniverse" application from https://github.com/IQSS/miniverse and give it read only access to your production Dataverse database. http://dataverse.org/metrics is powered by miniverse. - -Contents: +**Contents:** .. toctree:: + intro sword search dataaccess diff --git a/doc/sphinx-guides/source/api/intro.rst b/doc/sphinx-guides/source/api/intro.rst new file mode 100755 index 00000000000..d2f6c4e2487 --- /dev/null +++ b/doc/sphinx-guides/source/api/intro.rst @@ -0,0 +1,56 @@ +Introduction +============ + +We encourage anyone interested in building tools that interoperate with Dataverse to utilize our APIs. The Dataverse community has supplied :doc:`client-libraries` for Python, R, and Java and we are always interested in helping the community develop libraries for additional languages. The :doc:`apps` section links to open source Javascript, PHP, Python, and Java code that you can learn from while developing against Dataverse APIs. + +.. contents:: |toctitle| + :local: + +How This Guide is Organized +--------------------------- + +We document the Dataverse API in four sections: + +- :doc:`sword`: For depositing data using a standards-based approach rather than the :doc:`native-api`. +- :doc:`search`: For searching dataverses, datasets, and files. +- :doc:`dataaccess`: For downloading and subsetting data. +- :doc:`native-api`: For performing most tasks that are possible in the GUI. + +We use the term "native" to mean that the API is not based on any standard. For this reason, the :doc:`search` and :doc:`dataaccess` could also be considered "native" and in the future we may reorganize the API Guide to split the :doc:`native-api` section into "Datasets API", "Files" API, etc. + +Authentication +-------------- + +Most Dataverse APIs require the use of an API token. Instructions for getting a token are described in the :doc:`/user/account` section of the User Guide. + +There are two ways to pass your API token to Dataverse APIs. The preferred method is to send the token in the ``X-Dataverse-key`` HTTP header, as in the following curl example:: + + curl -H "X-Dataverse-key: 8b955f87-e49a-4462-945c-67d32e391e7e" https://demo.dataverse.org/api/datasets/:persistentId?persistentId=doi:TEST/12345 + +Throughout this guide you will often see Bash shell envionmental variables being used, like this:: + + export API_TOKEN='8b955f87-e49a-4462-945c-67d32e391e7e' + curl -H "X-Dataverse-key: $API_TOKEN" https://demo.dataverse.org/api/datasets/:persistentId?persistentId=doi:TEST/12345 + +The second way to pass your API token is via an extra query parameter called ``key`` in the URL like this:: + + curl "https://demo.dataverse.org/api/datasets/:persistentId?persistentId=doi:TEST/12345&key=$API_TOKEN" + +Use of the ``X-Dataverse-key`` HTTP header form is preferred because putting the query parameters in URLs often results in them finding their way into web server access logs. Your API token should be kept as secret as your password because it can be used to perform any action *as you* in the Dataverse application. + +Testing +------- + +Rather than using a production installation of Dataverse, API users are welcome to use http://demo.dataverse.org for testing. + +Support +------- + +If you are using the APIs for an installation of Dataverse hosted by your institution, you may want to reach out to the team that supports it. At the top of the Dataverse installation's home page, there should be a form you can fill out by clicking the "Support" link. + +If you are having trouble with http://demo.dataverse.org or have questions about the APIs, please feel free to reach out to the Dataverse community via https://groups.google.com/forum/#!forum/dataverse-community . + +Metrics +------- + +APIs described in this guide are shipped with the Dataverse software itself. Additional APIs are available if someone at your institution installs the "miniverse" application from https://github.com/IQSS/miniverse and gives it read only access to a production Dataverse database. http://dataverse.org/metrics is powered by miniverse. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index d41508d309c..adaff1ff237 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1,15 +1,16 @@ Native API ========== -Dataverse 4.0 exposes most of its GUI functionality via a REST-based API. Some API calls do not require authentication. Calls that do require authentication require the user's API key. That key can be passed either via an extra query parameter, ``key``, as in ``ENPOINT?key=API_KEY``, or via the HTTP header ``X-Dataverse-key``. Note that while the header option normally requires more work on client side, it is considered safer, as the API key is not logged in the server access logs. +Dataverse 4 exposes most of its GUI functionality via a REST-based API. This section describes that functionality. Most API endpoints require an API token that can be passed as the ``X-Dataverse-key`` HTTP header or in the URL as the ``key`` query parameter. .. note:: |CORS| Some API endpoint allow CORS_ (cross-origin resource sharing), which makes them usable from scripts runing in web browsers. These endpoints are marked with a *CORS* badge. .. _CORS: https://www.w3.org/TR/cors/ -.. warning:: Dataverse 4.0's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address//api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. +.. warning:: Dataverse 4's API is versioned at the URI - all API calls may include the version number like so: ``http://server-address/api/v1/...``. Omitting the ``v1`` part would default to the latest API version (currently 1). When writing scripts/applications that will be used for a long time, make sure to specify the API version, so they don't break when the API is upgraded. -.. contents:: +.. contents:: |toctitle| + :local: Endpoints --------- @@ -380,18 +381,27 @@ Example python code to replace a file. This may be run by changing these parame Builtin Users ~~~~~~~~~~~~~ -This endpoint deals with users of the built-in authentication provider. For more background on various authentication providers, see :doc:`/user/account` and :doc:`/installation/config`. +Builtin users are known as "Username/Email and Password" users in the :doc:`/user/account` of the User Guide. Dataverse stores a password (encrypted, of course) for these users, which differs from "remote" users such as Shibboleth or OAuth users where the password is stored elsewhere. See also "Auth Modes: Local vs. Remote vs. Both" in the :doc:`/installation/config` section of the Installation Guide. It's a valid configuration of Dataverse to not use builtin users at all. -For this service to work, the setting ``BuiltinUsers.KEY`` has to be set, and its value passed as ``key`` to -each of the calls. +Creating a Builtin User +^^^^^^^^^^^^^^^^^^^^^^^ -Generates a new user. Data about the user are posted via JSON. *Note that the password is passed as a parameter in the query*. :: +For security reasons, builtin users cannot be created via API unless the team who runs the Dataverse installation has populated a database setting called ``BuiltinUsers.KEY``, which is described under "Securing Your Installation" and "Database Settings" in the :doc:`/installation/config` section of the Installation Guide. You will need to know the value of ``BuiltinUsers.KEY`` before you can proceed. - POST http://$SERVER/api/builtin-users?password=$password&key=$key +To create a builtin user via API, you must first construct a JSON document. You can download :download:`user-add.json <../_static/api/user-add.json>` or copy the text below as a starting point and edit as necessary. -Gets the API token of the user, given the password. :: +.. literalinclude:: ../_static/api/user-add.json - GET http://$SERVER/api/builtin-users/$username/api-token?password=$password +Place this ``user-add.json`` file in your current directory and run the following curl command, substituting variables as necessary. Note that both the password of the new user and the value of ``BuiltinUsers.KEY`` are passed as query parameters:: + + curl -d @user-add.json -H "Content-type:application/json" "$SERVER_URL/api/builtin-users?password=$NEWUSER_PASSWORD&key=$BUILTIN_USERS_KEY" + +Retrieving the API Token of a Builtin User +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To retrieve the API token of a builtin user, given that user's password, use the curl command below:: + + curl "$SERVER_URL/api/builtin-users/$DV_USER_NAME/api-token?password=$DV_USER_PASSWORD" Roles ~~~~~ diff --git a/doc/sphinx-guides/source/api/search.rst b/doc/sphinx-guides/source/api/search.rst index 5ea6ff7a8c4..8a0524c2d43 100755 --- a/doc/sphinx-guides/source/api/search.rst +++ b/doc/sphinx-guides/source/api/search.rst @@ -1,10 +1,8 @@ Search API ========== -.. contents:: :local: - -About ------ +.. contents:: |toctitle| + :local: The Search API supports the same searching, sorting, and faceting operations as the Dataverse web interface. @@ -12,9 +10,13 @@ Unlike the web interface, this new API is limited to *published* data until `iss The parameters and JSON response are partly inspired by the `GitHub Search API `_. +.. note:: |CORS| The search API can be used from scripts running in web browsers, as it allows cross-origin resource sharing (CORS). + +.. _CORS: https://www.w3.org/TR/cors/ + + Please note that in Dataverse 4.3 and older the "citation" field wrapped the persistent ID URL in an ```` tag but this has been changed to plaintext. If you want the old value with HTML in it, a new field called "citationHtml" can be used. -The search API can be used from scripts running in web browsers, as it allows cross-origin resource sharing (CORS). Parameters ---------- @@ -264,3 +266,9 @@ Output from iteration example start: 10 total: 12 - Chestnut Sparrows (dataverse) - Wrens (dataverse) + +.. |CORS| raw:: html + + + CORS + diff --git a/doc/sphinx-guides/source/api/sword.rst b/doc/sphinx-guides/source/api/sword.rst index dbc4283395c..d894b27c35c 100755 --- a/doc/sphinx-guides/source/api/sword.rst +++ b/doc/sphinx-guides/source/api/sword.rst @@ -3,6 +3,12 @@ SWORD API SWORD_ stands for "Simple Web-service Offering Repository Deposit" and is a "profile" of AtomPub (`RFC 5023`_) which is a RESTful API that allows non-Dataverse software to deposit files and metadata into a Dataverse installation. :ref:`client-libraries` are available in Python, Java, R, Ruby, and PHP. +.. contents:: |toctitle| + :local: + +About +----- + Introduced in Dataverse Network (DVN) `3.6 `_, the SWORD API was formerly known as the "Data Deposit API" and ``data-deposit/v1`` appeared in the URLs. For backwards compatibility these URLs continue to work (with deprecation warnings). Due to architectural changes and security improvements (especially the introduction of API tokens) in Dataverse 4.0, a few backward incompatible changes were necessarily introduced and for this reason the version has been increased to ``v1.1``. For details, see :ref:`incompatible`. Dataverse implements most of SWORDv2_, which is specified at http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html . Please reference the `SWORDv2 specification`_ for expected HTTP status codes (i.e. 201, 204, 404, etc.), headers (i.e. "Location"), etc. For a quick introduction to SWORD, the two minute video at http://cottagelabs.com/news/intro-to-sword-2 is recommended. @@ -17,8 +23,6 @@ As a profile of AtomPub, XML is used throughout SWORD. As of Dataverse 4.0 datas .. _SWORDv2 specification: http://swordapp.github.io/SWORDv2-Profile/SWORDProfile.html -.. contents:: - .. _incompatible: Backward incompatible changes @@ -223,7 +227,7 @@ Client libraries - Python: https://github.com/swordapp/python-client-sword2 - Java: https://github.com/swordapp/JavaClient2.0 -- R: https://github.com/ropensci/dvn +- R: https://github.com/IQSS/dataverse-client-r - Ruby: https://github.com/swordapp/sword2ruby - PHP: https://github.com/swordapp/swordappv2-php-library diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 82cd679d93a..67e41844c03 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -64,9 +64,9 @@ # built documents. # # The short X.Y version. -version = '4.6.2' +version = '4.7' # The full version, including alpha/beta/rc tags. -release = '4.6.2' +release = '4.7' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -427,3 +427,7 @@ intersphinx_mapping = {'http://docs.python.org/': None} # Suppress "WARNING: unknown mimetype for ..." https://github.com/IQSS/dataverse/issues/3391 suppress_warnings = ['epub.unknown_project_files'] +rst_prolog = """ +.. |toctitle| replace:: Contents: +.. |anotherSub| replace:: Yes, there can be multiple. +""" diff --git a/doc/sphinx-guides/source/developers/branching-strategy.rst b/doc/sphinx-guides/source/developers/branching-strategy.rst deleted file mode 100755 index 05e0134a770..00000000000 --- a/doc/sphinx-guides/source/developers/branching-strategy.rst +++ /dev/null @@ -1,35 +0,0 @@ -================== -Branching Strategy -================== - -.. contents:: :local: - -Goals ------ - -The goals of the Dataverse branching strategy are: - -- allow for concurrent development -- only ship stable code - -We follow a simplified "git flow" model described at http://nvie.com/posts/a-successful-git-branching-model/ involving a "master" branch, a "develop" branch, and feature branches such as "1234-bug-fix". - -Branches --------- - -The "master" Branch -~~~~~~~~~~~~~~~~~~~ - -The "`master `_" branch represents released versions of Dataverse. As mentioned in the :doc:`making-releases` section, at release time we update the master branch to include all the code for that release. Commits are never made directly to master. Rather, master is updated only when we merge code into it from the "develop" branch. - -The "develop" Branch -~~~~~~~~~~~~~~~~~~~~ - -The "`develop `_" branch represents code that was stable enough to merge from a "feature" branch (described below) and that will make it into the next release. Like master, commits are never made to the develop branch. The develop branch is where integration occurs. Your goal is have your code merged into the develop branch after it has been reviewed. - -Feature Branches -~~~~~~~~~~~~~~~~ - -Feature branches are used for both developing features and fixing bugs. They are named after the GitHub issue they are meant to address, which means the branch should not be created until the GitHub issue exists. For example, if https://github.com/IQSS/dataverse/issues/1234 had a title of "Bug fix", you would name your branch "1234-bug-fix" or some other short "slug" with the issue number at the start. - -Always create your feature branch from the latest code in develop, pulling if necessary. Push your feature branch either to your fork of Dataverse or to the main repo at IQSS if you have write access. Create a pull request based on the feature branch you pushed. As mentioned in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md if you do not have access to advance your pull request into the "Code Review" column at https://waffle.io/IQSS/dataverse you should reach out to ask for it to be moved on your behalf. diff --git a/doc/sphinx-guides/source/developers/coding-style.rst b/doc/sphinx-guides/source/developers/coding-style.rst index 3334d927891..1a1cb6e33c1 100755 --- a/doc/sphinx-guides/source/developers/coding-style.rst +++ b/doc/sphinx-guides/source/developers/coding-style.rst @@ -2,8 +2,66 @@ Coding Style ============ -Like all development teams, the `Dataverse developers at IQSS `_ have their habits and styles when it comes to writing code. +Like all development teams, the `Dataverse developers at IQSS `_ have their habits and styles when it comes to writing code. Let's attempt to get on the same page. :) -A lot of it isn't written down, but a draft has been started at https://docs.google.com/document/d/1KTd3FpM1BI3HlBofaZjMmBiQEJtFf11jiiGpQeJzy7A/edit?usp=sharing +.. contents:: |toctitle| + :local: -Debate and comments are welcome! +Java +---- + +Formatting Code +~~~~~~~~~~~~~~~ + +Tabs vs. Spaces +^^^^^^^^^^^^^^^ + +Don't use tabs. Use spaces. + +Format Code You Changed with Netbeans +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As you probably gathered from the :doc:`dev-environment` section, IQSS has standardized on Netbeans. It is much appreciated when you format your code (but only the code you touched!) using the out-of-the-box Netbeans configuration. If you have created an entirely new Java class, you can just click Source -> Format. If you are adjusting code in an existing class, highlight the code you changed and then click Source -> Format. Keeping the "diff" in your pull requests small makes them easier to code review. + +We would like to someday automate the detection and possibly correction of code that hasn't been formatted using our house style (the default Netbeans style). We've heard that https://maven.apache.org/plugins/maven-checkstyle-plugin/ can do this but we would be happy to see a pull request in this area, especially if it also hooks up to our builds at https://travis-ci.org/IQSS/dataverse . + +Logging +~~~~~~~ + +We have adopted a pattern where the top of every class file has a line like this:: + + private static final Logger logger = Logger.getLogger(DatasetUtil.class.getCanonicalName()); + +Use this ``logger`` field with varying levels such as ``fine`` or ``info`` like this:: + + logger.fine("will get thumbnail from dataset logo"); + +Generally speaking you should use ``fine`` for everything that you don't want to show up in Glassfish's ``server.log`` file by default. If you use a higher level such as ``info`` for common operations, you will probably hear complaints that your code is too "chatty" in the logs. These logging levels can be controlled at runtime both on your development machine and in production as explained in the :doc:`debugging` section. + +When adding logging, do not simply add ``System.out.println()`` lines because the logging level cannot be controlled. + +Avoid Hard-Coding Strings +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Special strings should be defined as public constants. For example, ``DatasetFieldConstant.java`` contains a field for "title" and it's used in many places in the code (try "Find Usages" in Netbeans). This is better than writing the string "title" in all those places. + +Type Safety +~~~~~~~~~~~ + +If you just downloaded Netbeans and are using the out-of-the-box settings, you should be in pretty good shape. Unfortunately, the default configuration of Netbeans doesn't warn you about type-safety problems you may be inadvertently introducing into the code. To see these warnings, click Netbeans -> Preferences -> Editor -> Hints and check the following: + +- "Raw Types" under "Standard Javac Warnings" + +If you know of a way to easily share Netbeans configuration across a team, please get in touch. + + +Bike Shedding +------------- + +What color should the `bike shed `_ be? :) + +Come debate with us about coding style in this Google doc that has public comments enabled: https://docs.google.com/document/d/1KTd3FpM1BI3HlBofaZjMmBiQEJtFf11jiiGpQeJzy7A/edit?usp=sharing + +---- + +Previous: :doc:`debugging` | Next: :doc:`making-releases` diff --git a/doc/sphinx-guides/source/developers/debugging.rst b/doc/sphinx-guides/source/developers/debugging.rst index ea526778cda..e736cc29c81 100644 --- a/doc/sphinx-guides/source/developers/debugging.rst +++ b/doc/sphinx-guides/source/developers/debugging.rst @@ -2,7 +2,14 @@ Debugging ========= +.. contents:: |toctitle| + :local: + Logging ------- By default, Glassfish logs at the "INFO" level but logging can be increased to "FINE" on the fly with (for example) ``asadmin set-log-levels edu.harvard.iq.dataverse.api.Datasets=FINE``. Running ``asadmin list-log-levels`` will show the current logging levels. + +---- + +Previous: :doc:`documentation` | Next: :doc:`coding-style` diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 53317e4c459..4614f53b9d1 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -2,12 +2,13 @@ Development Environment ======================= -.. contents:: :local: +.. contents:: |toctitle| + :local: Assumptions ----------- -This guide assumes you are using a Mac. If you are using Windows or Linux, please reach out to other developers at https://groups.google.com/forum/#!forum/dataverse-dev +This guide assumes you are using a Mac. With some tweaks, it's not hard to get a dev environment set up on Linux. If you are using Windows, you might have the most success using Vagrant, which is listed under the :doc:`tools` section. Requirements ------------ @@ -26,7 +27,9 @@ Glassfish As a `Java Enterprise Edition `_ 7 (Java EE 7) application, Dataverse requires an applications server to run. -Glassfish 4.1 is required (not 4.1.1 until https://github.com/IQSS/dataverse/issues/2628 is resolved), which can be downloaded from http://glassfish.java.net . If you have downloaded Glassfish as part of a Netbeans bundle, you can manually add the proper version by clicking "Tools", "Servers", "Add Server". +Glassfish 4.1 is required (not any earlier or later versions until https://github.com/IQSS/dataverse/issues/2628 is resolved), which can be downloaded from http://download.oracle.com/glassfish/4.1/release/glassfish-4.1.zip . If you have downloaded Glassfish as part of a Netbeans bundle, you can manually add the proper version by clicking "Tools", "Servers", "Add Server". + +By default, Glassfish reports analytics information. The administration guide suggests this can be disabled with ``asadmin create-jvm-options -Dcom.sun.enterprise.tools.admingui.NO_NETWORK=true``, should this be found to be undesirable for development purposes. PostgreSQL ~~~~~~~~~~ @@ -58,12 +61,12 @@ Recommendations Mac OS X ~~~~~~~~ -The setup of a Dataverse development environment assumes the presence of a Unix shell (i.e. bash) so an operating system with Unix underpinnings such as Mac OS X or Linux is recommended. (The `development team at IQSS `_ has standardized Mac OS X.) Windows users are encouraged to install `Cygwin `_. +The setup of a Dataverse development environment assumes the presence of a Unix shell (i.e. bash) so an operating system with Unix underpinnings such as Mac OS X or Linux is recommended. (The `development team at IQSS `_ has standardized Mac OS X.) Windows users are encouraged to install `Cygwin `_. Netbeans ~~~~~~~~ -While developers are welcome to use any editor or IDE they wish, Netbeans 8+ is recommended because it is free of cost, works cross platform, has good support for Java EE projects, and happens to be the IDE that the `development team at IQSS `_ has standardized on. +While developers are welcome to use any editor or IDE they wish, Netbeans 8+ is recommended because it is free of cost, works cross platform, has good support for Java EE projects, and happens to be the IDE that the `development team at IQSS `_ has standardized on. NetBeans can be downloaded from http://netbeans.org. Please make sure that you use an option that contains the Jave EE features when choosing your download bundle. While using the installer you might be prompted about installing JUnit and Glassfish. There is no need to reinstall Glassfish, but it is recommended that you install JUnit. @@ -91,12 +94,12 @@ From the terminal, ``ssh-keygen`` will create new ssh keys for you: Clone Project from GitHub ~~~~~~~~~~~~~~~~~~~~~~~~~ -Before making commits, please read about our :doc:`/developers/branching-strategy` to make sure you commit to the right branch. +Before cloning the repo, you are invited to read about our branching strategy in the :doc:`version-control` section but we'll explain the basics here. Determine Which Repo To Push To ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Developers who are not part of the `development team at IQSS `_ should first fork https://github.com/IQSS/dataverse per https://help.github.com/articles/fork-a-repo/ +Developers who are not part of the `development team at IQSS `_ should first fork https://github.com/IQSS/dataverse per https://help.github.com/articles/fork-a-repo/ Cloning the Project from Netbeans ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -116,6 +119,25 @@ If you do have push access to https://github.com/IQSS/dataverse clone it: ``git clone git@github.com:IQSS/dataverse.git`` +Building the WAR File +~~~~~~~~~~~~~~~~~~~~~ + +Soon, we'll be running the Dataverse installer, but before we do, we must build the Dataverse application, which is delivered as a "WAR" file. WAR stands for "Web application ARchive" and you can read more about this packaging format at https://en.wikipedia.org/wiki/WAR_(file_format) + +The first time you build the war file, it may take a few minutes while dependencies are downloaded from Maven Central. + +We'll describe below how to build the WAR file from both Netbean and the terminal, but in both cases, you'll want to see the output "BUILD SUCCESS". + +Building the War File from Netbeans +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +From Netbeans, click "Run" and then "Build Project (dataverse)". + +Building the War File from the Terminal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +After cloning the git repo, you need to ``cd`` into ``dataverse`` and run ``mvn package``. If you don't have the ``mvn`` command available to you, you need to install Maven, which is mentioned in the :doc:`tools` section. + Installing and Running Solr ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -138,7 +160,12 @@ Once some dataverses, datasets, and files have been created and indexed, you can Run Installer ~~~~~~~~~~~~~ -Once you install Glassfish and PostgreSQL, you need to configure the environment for the Dataverse app - configure the database connection, set some options, etc. We have a new installer script that should do it all for you. Again, assuming that the clone on the Dataverse repository was retrieved using NetBeans and that it is saved in the path ~/NetBeansProjects: +Please note the following: + +- If you have trouble with the SMTP server, consider editing the installer script to disable the SMTP check. +- Rather than running the installer in "interactive" mode, it's possible to put the values in a file. See "non-interactive mode" in the :doc:`/installation/installation-main` section of the Installation Guide. + +Now that you have all the prerequisites in place, you need to configure the environment for the Dataverse app - configure the database connection, set some options, etc. We have an installer script that should do it all for you. Again, assuming that the clone on the Dataverse repository was retrieved using NetBeans and that it is saved in the path ~/NetBeansProjects: ``cd ~/NetBeansProjects/dataverse/scripts/installer`` @@ -150,6 +177,50 @@ The script is a variation of the old installer from DVN 3.x that calls another s All the future changes to the configuration that are Glassfish-specific and can be done through ``asadmin`` should now go into ``scripts/install/glassfish-setup.sh``. +FIXME: Add a "dev" mode to the installer to allow REST Assured tests to be run. For now, refer to the steps in the :doc:`testing` section. + +Iterating on Code and Redeploying +--------------------------------- + +Deploy on Save +~~~~~~~~~~~~~~ + +Out of the box, Netbeans is configured to "Deploy on Save" which means that if you save any changes to project files such as Java classes, XHTML files, or "bundle" files (i.e. Bundle.properties), the project is recompiled and redeployed to Glassfish automatically. This behavior works well for many of us but if you don't like it, you can turn it off by right-clicking "dataverse" under the Projects tab, clicking "Run" and unchecking "Deploy on Save". + +Deploying Manually +~~~~~~~~~~~~~~~~~~ + +For developers not using Netbeans, or deploying to a non-local system for development, code can be deployed manually. +There are four steps to this process: + +1. Build the war file: ``mvn package`` +2. Undeploy the Dataverse application (if necessary): ``asadmin undeploy dataverse-VERSION`` +3. Copy the war file to the development server (if necessary) +4. Deploy the new code: ``asadmin deploy /path/to/dataverse-VERSION.war`` + +The :doc:`/installation/installation-main` section of the Installation Guide has more information on this topic. + +Netbeans Connector Chrome Extension +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For faster iteration while working on JSF pages, it is highly recommended that you install the Netbeans Connector Chrome Extension listed in the :doc:`tools` section. When you save XHTML or CSS files, you will see the changes immediately. + +Troubleshooting +--------------- + +We've described above the "happy path" of when everything goes right with setting up your Dataverse development environment. Here are some common problems and solutions for when things go wrong. + +context-root in glassfish-web.xml Munged by Netbeans +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For unknown reasons, Netbeans will sometimes change the following line under ``src/main/webapp/WEB-INF/glassfish-web.xml``: + +``/`` + +Sometimes Netbeans will change ``/`` to ``/dataverse``. Sometimes it will delete the line entirely. Either way, you will see very strange behavior when attempting to click around Dataverse in a browser. The home page will load but icons will be missing. Any other page will fail to load entirely and you'll see a Glassfish error. + +The solution is to put the file back to how it was before Netbeans touched it. If anyone knows of an open Netbeans bug about this, please let us know. + Rebuilding Your Dev Environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -193,3 +264,7 @@ Geoconnect Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. To set up a Geoconnect development environment, you can follow the steps outlined in the `local_setup.md `_ guide. You will need Python and a few other prerequisites. As mentioned under "Architecture and Components" in the :doc:`/installation/prep` section of the Installation Guide, Geoconnect is an optional component of Dataverse, so this section is only necessary to follow it you are working on an issue related to this feature. + +---- + +Previous: :doc:`intro` | Next: :doc:`version-control` diff --git a/doc/sphinx-guides/source/developers/documentation.rst b/doc/sphinx-guides/source/developers/documentation.rst index 1908dc59ba5..fcca7aa0558 100755 --- a/doc/sphinx-guides/source/developers/documentation.rst +++ b/doc/sphinx-guides/source/developers/documentation.rst @@ -2,6 +2,9 @@ Documentation ============= +.. contents:: |toctitle| + :local: + Quick Fix ----------- @@ -61,3 +64,17 @@ After sphinx is done processing the files you should notice that the html folder You can click on the files in the html folder to preview the changes. Now you can make a commit with the changes to your own fork in GitHub and submit a pull request to the dataverse repository. + +Table of Contents +----------------- + +Every non-index page should use the following code to display a table of contents of internal sub-headings: :: + + .. contents:: |toctitle| + :local: + +This code should be placed below any introductory text/images and directly above the first subheading, much like a Wikipedia page. + +---- + +Previous: :doc:`testing` | Next: :doc:`debugging` diff --git a/doc/sphinx-guides/source/developers/geospatial.rst b/doc/sphinx-guides/source/developers/geospatial.rst index 55fbf9e6808..45d2cd3a368 100644 --- a/doc/sphinx-guides/source/developers/geospatial.rst +++ b/doc/sphinx-guides/source/developers/geospatial.rst @@ -2,6 +2,9 @@ Geospatial Data =============== +.. contents:: |toctitle| + :local: + How Dataverse Ingests Shapefiles -------------------------------- @@ -171,4 +174,8 @@ The ``get_join_targets()`` function in ``dataverse_layer_services.py`` uses the Saving Join Target Information to Geoconnect Database ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``get_latest_jointarget_information()`` in ``utils.py`` retrieves recent JoinTarget Information from the database. (See the `utils code in GitHub `_.) \ No newline at end of file +The ``get_latest_jointarget_information()`` in ``utils.py`` retrieves recent JoinTarget Information from the database. (See the `utils code in GitHub `_.) + +---- + +Previous: :doc:`unf/index` | Next: :doc:`selinux` diff --git a/doc/sphinx-guides/source/developers/index.rst b/doc/sphinx-guides/source/developers/index.rst index f37afbbd173..461af55864d 100755 --- a/doc/sphinx-guides/source/developers/index.rst +++ b/doc/sphinx-guides/source/developers/index.rst @@ -6,13 +6,13 @@ Developer Guide ======================================================= -Contents: +**Contents:** .. toctree:: intro dev-environment - branching-strategy + version-control testing documentation debugging diff --git a/doc/sphinx-guides/source/developers/intro.rst b/doc/sphinx-guides/source/developers/intro.rst index fa25d14b67e..633297e74b7 100755 --- a/doc/sphinx-guides/source/developers/intro.rst +++ b/doc/sphinx-guides/source/developers/intro.rst @@ -4,12 +4,15 @@ Introduction Welcome! `Dataverse `_ is an `open source `_ project that loves `contributors `_! +.. contents:: |toctitle| + :local: + Intended Audience ----------------- This guide is intended primarily for developers who want to work on the main Dataverse code base at https://github.com/IQSS/dataverse -To get started, you'll want to set up your :doc:`/developers/dev-environment` and make sure you understand the :doc:`/developers/branching-strategy`. Thorough :doc:`/developers/testing` is encouraged (and expected). Opinions about :doc:`/developers/coding-style` are welcome! +To get started, you'll want to set up your :doc:`dev-environment` and make sure you understand the branching strategy described in the :doc:`version-control` section. :doc:`testing` is expected. Opinions about :doc:`coding-style` are welcome! If you have any questions at all, please reach out to other developers per https://github.com/IQSS/dataverse/blob/master/CONTRIBUTING.md @@ -36,11 +39,17 @@ Related Projects As a developer, you also may be interested in these projects related to Dataverse: +- Miniverse - expose metrics from a Dataverse database: https://github.com/IQSS/miniverse - `Zelig `_ (R) - run statistical models on files uploaded to Dataverse: https://github.com/IQSS/Zelig -- `TwoRavens `_ (Javascript) - a `d3.js `_ interface for exploring data and running Zelig models: https://github.com/IQSS/TwoRavens +- `TwoRavens `_ (Javascript) - a `d3.js `_ interface for exploring data and running Zelig models: https://github.com/IQSS/TwoRavens - :doc:`/developers/unf/index` (Java) - a Universal Numerical Fingerprint: https://github.com/IQSS/UNF - `DataTags `_ (Java and Scala) - tag datasets with privacy levels: https://github.com/IQSS/DataTags - GeoConnect (Python) - create a map by uploading files to Dataverse: https://github.com/IQSS/geoconnect - Dataverse API client libraries - use Dataverse APIs from various languages: :doc:`/api/client-libraries` - Third party apps - make use of Dataverse APIs: :doc:`/api/apps` +- chat.dataverse.org - chat interface for Dataverse users and developers: https://github.com/IQSS/chat.dataverse.org - [Your project here] :) + +---- + +Next: :doc:`dev-environment` diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index d690394cad9..23cc5c93e45 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -2,7 +2,8 @@ Making Releases =============== -.. contents:: :local: +.. contents:: |toctitle| + :local: Bump Version Numbers -------------------- @@ -18,7 +19,7 @@ Here's an example commit where all three files were updated at once: https://git Merge "develop" into "master" ----------------------------- -The "develop" branch should be merged into "master" before tagging. See also :doc:`branching-strategy`. +The "develop" branch should be merged into "master" before tagging. See also the branching strategy described in the :doc:`version-control` section. Write Release Notes ------------------- @@ -42,3 +43,7 @@ Publish Release --------------- Click the "Publish release" button. + +---- + +Previous: :doc:`coding-style` | Next: :doc:`tools` diff --git a/doc/sphinx-guides/source/developers/selinux.rst b/doc/sphinx-guides/source/developers/selinux.rst index 7062a7d1f01..582510c5847 100644 --- a/doc/sphinx-guides/source/developers/selinux.rst +++ b/doc/sphinx-guides/source/developers/selinux.rst @@ -2,7 +2,8 @@ SELinux ======= -.. contents:: :local: +.. contents:: |toctitle| + :local: Introduction ------------ @@ -108,3 +109,7 @@ Once your updated SELinux rules are in place, try logging in with Shibboleth aga Keep iterating until it works and then create a pull request based on your updated file. Good luck! Many thanks to Bill Horka from IQSS for his assistance in explaining how to construct a SELinux Type Enforcement (TE) file! + +---- + +Previous: :doc:`geospatial` diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index a79d2d08570..229ed63ac3e 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -2,12 +2,221 @@ Testing ======= +In order to keep our codebase healthy, the Dataverse project encourages developers to write automated tests in the form of unit tests and integration tests. We also welcome ideas for how to improve our automated testing. + +.. contents:: |toctitle| + :local: + +The Health of a Codebase +------------------------ + +Before we dive into the nut and bolts of testing, let's back up for a moment and think about why we write automated tests in the first place. Writing automated tests is an investment and leads to better quality software. Counterintuitively, writing tests and executing them regularly allows a project to move faster. Martin Fowler explains this well while talking about the health of a codebase: + + "This is an economic judgment. Several times, many times, I run into teams that say something like, 'Oh well. Management isn't allowing us to do a quality job here because it will slow us down. And we've appealed to management and said we need to put more quality in the code, but they've said no, we need to go faster instead.' And my comment to that is well, as soon as you’re framing it in terms of code quality versus speed, you've lost. Because the whole point of refactoring is to go faster. + + "And this is why I quite like playing a bit more with the metaphor as the health of a codebase. If you keep yourself healthy then you'll be able to run faster. But if you just say, 'Well, I want to run a lot so I'm therefore going to run a whole load all the time and not eat properly and not pay attention about this shooting pain going up my leg,' then you’re not going to be able to run quickly very long. **You have to pay attention to your health. And same with the codebase. You have to continuously say, 'How do we keep it in a healthy state? Then we can go fast,' because we’re running marathons here with codebases. And if we neglect that internal quality of the codebase, it hits you surprisingly fast.**" + + --Martin Fowler at https://devchat.tv/ruby-rogues/178-rr-book-club-refactoring-ruby-with-martin-fowler + +Testing in Depth +---------------- + +`Security in depth `_ might mean that your castle has a moat as well as high walls. Likewise, when testing, you should consider testing a various layers of the stack using both unit tests and integration tests. + +When writing tests, you may find it helpful to first map out which functions of your code you want to test, and then write a functional unit test for each which can later comprise a larger integration test. + Unit Tests ---------- -Write them. JUnit is your friend. +Creating unit tests for your code is a helpful way to test what you've built piece by piece. + +Unit tests can be executed without runtime dependencies on PostgreSQL, Solr, or any other external system. They are the lowest level of testing and are executed constantly on developers' laptops as part of the build process and via continous integration services in the cloud. + +A unit test should execute an operation of your code in a controlled fashion. You must make an assertion of what the expected response gives back. It's important to test optimistic output and assertions (the "happy path"), as well as unexpected input that leads to failure conditions. Know how your program should handle anticipated errors/exceptions and confirm with your test(s) that it does so properly. + +Unit Test Automation Overview +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We use a variety of tools to write, execute, and measure the code coverage of unit tests, including Maven, JUnit, Jacoco, GitHub, Travis, and Coveralls. We'll explain the role of each tool below, but here's an overview of what you can expect from the automation we've set up. + +As you prepare to make a pull request, as described in the :doc:`version-control` section, you will be working on a new branch you create from the "develop" branch. Let's say your branch is called ``1012-private-url``. As you work, you are constantly invoking Maven to build the war file. When you do a "clean and build" in Netbeans, Maven runs all the unit tests (anything ending with ``Test.java``) and the runs the results through a tool called Jacoco that calculates code coverage. When you push your branch to GitHub and make a pull request, a web service called Travis CI runs Maven and Jacoco on your branch and pushes the results to Coveralls, which is a web service that tracks changes to code coverage over time. + +To make this more concrete, observe that https://github.com/IQSS/dataverse/pull/3111 has comments from a GitHub user called ``coveralls`` saying things like "Coverage increased (+0.5%) to 5.547% when pulling dd6ceb1 on 1012-private-url into be5b26e on develop." Clicking on the comment should lead you to a URL such as https://coveralls.io/builds/7013870 which shows how code coverage has gone up or down. That page links to a page such as https://travis-ci.org/IQSS/dataverse/builds/144840165 which shows the build on the Travis side that pushed the results ton Coveralls. + +The main takeaway should be that we care about unit testing enough to measure the changes to code coverage over time using automation. Now let's talk about how you can help keep our code coverage up by writing unit tests with JUnit. + +Writing Unit Tests with JUnit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +We are aware that there are newer testing tools such as TestNG, but we use `JUnit `_ because it's tried and true. + +If writing tests is new to you, poke around existing unit tests which all end in ``Test.java`` and live under ``src/test``. Each test is annotated with ``@Test`` and should have at least one assertion which specifies the expected result. In Netbeans, you can run all the tests in it by clicking "Run" -> "Test File". From the test file, you should be able to navigate to the code that's being tested by right-clicking on the file and clicking "Navigate" -> "Go to Test/Tested class". Likewise, from the code, you should be able to use the same "Navigate" menu to go to the tests. + +Refactoring Code to Make It Unit-Testable +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Existing code is not necessarily written in a way that lends itself to easy testing. Generally speaking, it is difficult to write unit tests for both JSF "backing" beans (which end in ``Page.java``) and "service" beans (which end in ``Service.java``) because they require the database to be running in order to test them. If service beans can be exercised via API they can be tested with integration tests (described below) but a good technique for making the logic testable it to move code to "util beans" (which end in ``Util.java``) that operate on Plain Old Java Objects (POJOs). ``PrivateUrlUtil.java`` is a good example of moving logic from ``PrivateUrlServiceBean.java`` to a "util" bean to make the code testable. + +Observing Changes to Code Coverage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once you've written some tests, you're probably wondering how much you've helped to increase the code coverage. In Netbeans, do a "clean and build." Then, under the "Projects" tab, right-click "dataverse" and click "Code Coverage" -> "Show Report". For each Java file you have open, you should be able to see the percentage of code that is covered by tests and every line in the file should be either green or red. Green indicates that the line is being exercised by a unit test and red indicates that it is not. + +In addition to seeing code coverage in Netbeans, you can also see code coverage reports by opening ``target/site/jacoco/index.html`` in your browser. + +Testing Commands +^^^^^^^^^^^^^^^^ + +You might find studying the following test classes helpful in writing tests for commands: + +- CreatePrivateUrlCommandTest.java +- DeletePrivateUrlCommandTest.java +- GetPrivateUrlCommandTest.java + +In addition, there is a writeup on "The Testable Command" at https://github.com/IQSS/dataverse/blob/develop/doc/theTestableCommand/TheTestableCommand.md . Integration Tests ----------------- -Write them. `REST-assured `_ and `Selenium `_ are recommended. A Jenkins environment is available for automated testing: https://build.hmdc.harvard.edu:8443 +Unit tests are fantastic for low level testing of logic but aren't especially real-world-applicable because they do not exercise Dataverse as it runs in production with a database and other runtime dependencies. We test in-depth by also writing integration tests to exercise a running system. + +Unfortunately, the term "integration tests" can mean different things to different people. For our purposes, an integration test has the following qualities: + +- Integration tests exercise Dataverse APIs. +- Integration tests are not automatically on developers' laptops. +- Integration tests operate on an installation of Dataverse that is running and able to talk to both PostgreSQL and Solr. +- Integration tests are written using REST Assured. + +Getting Set Up to Run REST Assured Tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Unit tests are run automatically on every build, but dev environments and servers require special setup to run REST Assured tests. In short, Dataverse needs to be placed into an insecure mode that allows arbitrary users and datasets to be created and destroyed. This differs greatly from the out-of-the-box behavior of Dataverse, which we strive to keep secure for sysadmins installing the software for their institutions in a production environment. + +The :doc:`dev-environment` section currently refers developers here for advice on getting set up to run REST Assured tests, but we'd like to add some sort of "dev" flag to the installer to put Dataverse in "insecure" mode, with lots of scary warnings that this dev mode should not be used in production. + +The Burrito Key +^^^^^^^^^^^^^^^ + +For reasons that have been lost to the mists of time, Dataverse really wants you to to have a burrito. Specifically, if you're trying to run REST Assured tests and see the error "Dataverse config issue: No API key defined for built in user management", you must run the following curl command (or make an equivalent change to your database): + +``curl -X PUT -d 'burrito' http://localhost:8080/api/admin/settings/BuiltinUsers.KEY`` + +Without this "burrito" key in place, REST Assured will not be able to create users. We create users to create objects we want to test, such as dataverses, datasets, and files. + +Root Dataverse Permissions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In your browser, log in as dataverseAdmin (password: admin) and click the "Edit" button for your root dataverse. Navigate to Permissions, then the Edit Access button. Under "Who can add to this dataverse?" choose "Anyone with a dataverse account can add sub dataverses" if it isn't set to this already. + +Alternatively, this same step can be done with this script: ``scripts/search/tests/grant-authusers-add-on-root`` + +Publish Root Dataverse +^^^^^^^^^^^^^^^^^^^^^^ + +The root dataverse must be published for some of the REST Assured tests to run. + +dataverse.siteUrl +^^^^^^^^^^^^^^^^^ + +When run locally (as opposed to a remote server), some of the REST Assured tests require the ``dataverse.siteUrl`` JVM option to be set to ``http://localhost:8080``. See "JVM Options" under the :doc:`/installation/config` section of the Installation Guide for advice changing JVM options. First you should check to check your JVM options with: + +``asadmin list-jvm-options | egrep 'dataverse|doi'`` + +If ``dataverse.siteUrl`` is absent, you can add it with: + +``asadmin create-jvm-options "-Ddataverse.siteUrl=http\://localhost\:8080"`` + +Identifier Generation +^^^^^^^^^^^^^^^^^^^^^ + +``DatasetsIT.java`` exercises the feature where the "identifier" of a DOI can be a digit and requires a sequence to be added to your database. See ``:IdentifierGenerationStyle`` under the :doc:`/installation/config` section for adding this sequence to your installation of PostgreSQL. + + +Writing Integration Tests with REST Assured +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Before writing any new REST Assured tests, you should get the tests to pass in an existing REST Assured test file. ``BuiltinUsersIT.java`` is relatively small and requires less setup than other test files. + +You do not have to reinvent the wheel. There are many useful methods you can call in your own tests -- especially within UtilIT.java -- when you need your test to create and/or interact with generated accounts, files, datasets, etc. Similar methods can subsequently delete them to get them out of your way as desired before the test has concluded. + +For example, if you’re testing your code’s operations with user accounts, the method ``UtilIT.createRandomUser();`` can generate an account for your test to work with. The same account can then be deleted by your program by calling the ``UtilIT.deleteUser();`` method on the imaginary friend your test generated. + +Remember, it’s only a test (and it's not graded)! Some guidelines to bear in mind: + +- Map out which logical functions you want to test +- Understand what’s being tested and ensure it’s repeatable +- Assert the conditions of success / return values for each operation + * A useful resource would be `HTTP status codes `_ +- Let the code do the labor; automate everything that happens when you run your test file. +- Just as with any development, if you’re stuck: ask for help! + +To execute existing integration tests on your local Dataverse, a helpful command line tool to use is `Maven `_. You should have Maven installed as per the `Development Environment `_ guide, but if not it’s easily done via Homebrew: ``brew install maven``. + +Once installed, you may run commands with ``mvn [options] [] []``. + ++ If you want to run just one particular API test, it’s as easy as you think: + + ``mvn test -Dtest=FileRecordJobIT`` + ++ To run more than one test at a time, separate by commas: + + ``mvn test -Dtest=FileRecordJobIT,ConfirmEmailIT`` + ++ To run any test(s) on a particular domain, replace localhost:8080 with desired domain name: + + ``mvn test -Dtest=FileMetadataIT -Ddataverse.test.baseurl='http://localhost:8080'`` + +The Phoenix Server +------------------ + +A server at http://phoenix.dataverse.org has been set up to test the latest code from the develop branch. Testing is done using chained builds of Jenkins jobs: + +- A war file is built from the latest code in develop: https://build.hmdc.harvard.edu:8443/job/phoenix.dataverse.org-build-develop/ +- The resulting war file is depoyed to the Phoenix server: https://build.hmdc.harvard.edu:8443/job/phoenix.dataverse.org-deploy-develop/ +- REST Assured Tests are run across the wire from the Jenkins server to the Phoenix server: https://build.hmdc.harvard.edu:8443/job/phoenix.dataverse.org-apitest-develop/ + +Future Work +----------- + +We'd like to make improvements to our automated testing. See also 'this thread from our mailing list '_ asking for ideas from the community, and discussion at 'this GitHub issue. '_ + +Future Work on Unit Tests +~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Review pull requests from @bencomp for ideas for approaches to testing: https://github.com/IQSS/dataverse/pulls?q=is%3Apr+author%3Abencomp +- Come up with a way to test commands: http://irclog.iq.harvard.edu/dataverse/2015-11-04#i_26750 +- Test EJBs using Arquillian, embedded Glassfish, or similar. @bmckinney kicked the tires on Arquillian at https://github.com/bmckinney/bio-dataverse/commit/2f243b1db1ca704a42cd0a5de329083763b7c37a + +Future Work on Integration Tests +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- Automate testing of the Python client: https://github.com/IQSS/dataverse-client-python/issues/10 +- Work with @leeper on testing the R client: https://github.com/IQSS/dataverse-client-r +- Review and attempt to implement "API Test Checklist" from @kcondon at https://docs.google.com/document/d/199Oq1YwQ4pYCguaeW48bIN28QAitSk63NbPYxJHCCAE/edit?usp=sharing +- Attempt to use @openscholar approach for running integration tests using Travis https://github.com/openscholar/openscholar/blob/SCHOLAR-3.x/.travis.yml (probably requires using Ubuntu rather than CentOS) +- Generate code coverage reports for **integration** tests: https://github.com/pkainulainen/maven-examples/issues/3 and http://www.petrikainulainen.net/programming/maven/creating-code-coverage-reports-for-unit-and-integration-tests-with-the-jacoco-maven-plugin/ +- Consistent logging of API Tests. Show test name at the beginning and end and status codes returned. + +Browser-Based Testing +~~~~~~~~~~~~~~~~~~~~~ + +- Revisit Selenium/Open Sauce: https://github.com/IQSS/dataverse/commit/8a26404 and https://saucelabs.com/u/esodvn and https://saucelabs.com/u/wdjs and http://sauceio.com/index.php/2013/05/a-browser-matrix-widget-for-the-open-source-community/ + +Installation Testing +~~~~~~~~~~~~~~~~~~~~ + +- Run `vagrant up` on a server to test the installer: http://guides.dataverse.org/en/latest/developers/tools.html#vagrant . We haven't been able to get this working in Travis: https://travis-ci.org/IQSS/dataverse/builds/96292683 . Perhaps it would be possible to use AWS as a provider from Vagrant judging from https://circleci.com/gh/critical-alert/circleci-vagrant/6 . +- Work with @lwo to automate testing of https://github.com/IQSS/dataverse-puppet . Consider using Travis: https://github.com/IQSS/dataverse-puppet/issues/10 +- Work with @donsizemore to automate testing of https://github.com/IQSS/dataverse-ansible with Travis or similar. + +Load/Performance Testing +~~~~~~~~~~~~~~~~~~~~~~~~ + +- Run stress tests on a period basis: https://github.com/IQSS/dataverse-helper-scripts/tree/master/src/stress_tests +- Marcel Duran created a command-line wrapper for the WebPagetest API that can be used to test performance in your continuous integration pipeline (TAP, Jenkins, Travis-CI, etc): https://github.com/marcelduran/webpagetest-api/wiki/Test-Specs#jenkins-integration +- Documentation +- Create top-down checklist, building off the spreadsheet at https://github.com/IQSS/dataverse/issues/3358#issuecomment-256400776 + +---- + +Previous: :doc:`version-control` | Next: :doc:`documentation` diff --git a/doc/sphinx-guides/source/developers/tools.rst b/doc/sphinx-guides/source/developers/tools.rst index ab18e8b87f3..e1d2f5b9fa0 100755 --- a/doc/sphinx-guides/source/developers/tools.rst +++ b/doc/sphinx-guides/source/developers/tools.rst @@ -4,7 +4,8 @@ Tools These are handy tools for your :doc:`/developers/dev-environment/`. -.. contents:: :local: +.. contents:: |toctitle| + :local: Netbeans Connector Chrome Extension +++++++++++++++++++++++++++++++++++ @@ -66,6 +67,8 @@ Vagrantfile). Please note that running ``vagrant up`` for the first time should run the ``downloads/download.sh`` script for you to download required software such as Glassfish and Solr and any patches. However, these dependencies change over time so it's a place to look if ``vagrant up`` was working but later fails. +On Windows if you see an error like ``/usr/bin/perl^M: bad interpreter`` you might need to run ``dos2unix`` on the installation scripts. + MSV +++ @@ -77,3 +80,7 @@ MSV start parsing a grammar. validating ddi.xml the document is valid. + +---- + +Previous: :doc:`making-releases` | Next: :doc:`unf/index` diff --git a/doc/sphinx-guides/source/developers/unf/index.rst b/doc/sphinx-guides/source/developers/unf/index.rst index dc2f37d0ba9..0651891242d 100644 --- a/doc/sphinx-guides/source/developers/unf/index.rst +++ b/doc/sphinx-guides/source/developers/unf/index.rst @@ -4,15 +4,6 @@ Universal Numerical Fingerprint (UNF) ===================================== -Contents: - -.. toctree:: - :maxdepth: 2 - - unf-v3 - unf-v5 - unf-v6 - .. figure:: ./img/unf-diagram.png :align: center :alt: alternate text @@ -38,5 +29,18 @@ available, for cross-validation. Learn more: Micah Altman, Jeff Gill and Michael McDonald, 2003, `Numerical Issues in Statistical Computing for the Social Scientist -`_, +`_, New York: John Wiley. + +**Contents:** + +.. toctree:: + :maxdepth: 2 + + unf-v3 + unf-v5 + unf-v6 + +---- + +Previous: :doc:`/developers/tools` | Next: :doc:`/developers/geospatial` diff --git a/doc/sphinx-guides/source/developers/unf/unf-v3.rst b/doc/sphinx-guides/source/developers/unf/unf-v3.rst index 2bb6f64a6f2..3f0018d7fa5 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v3.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v3.rst @@ -3,6 +3,9 @@ UNF Version 3 =========================== +.. contents:: |toctitle| + :local: + Version 3 of the UNF algorithm was used by the Dataverse Network software prior to version 2.0, and was implemented in R code. This algorithm was used on digital objects containing vectors of numbers, vectors of character strings, data sets comprising such vectors, and studies comprising one or more such data sets. The UNF V3 algorithm applied to the content of a data set or study is as follows: diff --git a/doc/sphinx-guides/source/developers/unf/unf-v5.rst b/doc/sphinx-guides/source/developers/unf/unf-v5.rst index 4fb160c20ea..29b4556b1f9 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v5.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v5.rst @@ -3,10 +3,13 @@ UNF Version 5 ================================ +.. contents:: |toctitle| + :local: + **Important Update:** -**UNF Version 5 has been in use by the Dataverse project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours.** +UNF Version 5 has been in use by the Dataverse project since 2009. It was built into every version of the DVN, starting with 2.0 and up to 3.6.2. However, some problems were recently found in that implementation. Namely, in certain cases data normalization is not implemented fully to the spec. UNF signatures it generates are still reasonably strong statistically; however, this means that at least some of our signatures are not independently verifiable. I.e., if somebody fully implements their own version of UNF calculator, for certain datasets it would calculate signatures different from those generated by the DVN. Unless of course they implement it with the exact same bugs as ours. -**To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation.** +To address this, the Project is about to release UNF Version 6. The release date is still being discussed. It may coincide with the release of Dataverse 4.0. Alternatively, the production version of DVN 3.6.3 may get upgraded to use UNF v6 prior to that. This will be announced shortly. In the process, we are solving another problem with UNF v5 - this time we've made an effort to offer very implementer-friendly documentation that describes the algorithm fully and unambiguously. So if you are interested in implementing your own version of a UNF calculator, (something we would like to encourage!) please proceed directly to the Version 6 documentation. -**Going forward, we are going to offer a preserved version of the Version 5 library and, possibly, an online UNF v5 calculator, for the purposes of validating vectors and data sets for which published Version 5 UNFs exist.** +Going forward, we are going to offer a preserved version of the Version 5 library and, possibly, an online UNF v5 calculator, for the purposes of validating vectors and data sets for which published Version 5 UNFs exist. diff --git a/doc/sphinx-guides/source/developers/unf/unf-v6.rst b/doc/sphinx-guides/source/developers/unf/unf-v6.rst index 2a67e5255f6..b5c898a9a08 100644 --- a/doc/sphinx-guides/source/developers/unf/unf-v6.rst +++ b/doc/sphinx-guides/source/developers/unf/unf-v6.rst @@ -3,6 +3,8 @@ UNF Version 6 ================================ + + *(this document is a draft!)* The document is primarily intended for those who are interested in implementing their own UNF Version 6 calculator. We would like to encourage multiple parallel implementations, since that would be a great (the only, really) way to cross-validate UNF signatures calculated for specific sets of data. @@ -11,6 +13,9 @@ The document is primarily intended for those who are interested in implementing UNF v5, on which v6 is based, was originally described in Dr. Micah Altman's paper "A Fingerprint Method for Verification of Scientific Data", Springer Verlag, 2008. The reader is encouraged to consult it for the explanation of the theory behind UNF. However, various changes and clarifications concerning the specifics of normalization have been made to the algorithm since the publication. These crucial details were only documented in the author's unpublished edits of the article and in private correspondence. With this document, a serious effort has been made to produce a complete step-by-step description of the entire process. It should be fully sufficient for the purposes of implementing the algorithm. +.. contents:: |toctitle| + :local: + I. UNF of a Data Vector ------------------------- diff --git a/doc/sphinx-guides/source/developers/version-control.rst b/doc/sphinx-guides/source/developers/version-control.rst new file mode 100644 index 00000000000..f95054eee87 --- /dev/null +++ b/doc/sphinx-guides/source/developers/version-control.rst @@ -0,0 +1,130 @@ +================== +Version Control +================== + +The Dataverse Project uses git for version control and GitHub for hosting. On this page we'll explain where to find the code, our branching strategey, advice on how to make a pull request, and other git tips. + +.. contents:: |toctitle| + :local: + + +Where to Find the Dataverse Code +-------------------------------- + +The main Dataverse code at https://github.com/IQSS/dataverse but as explained in the :doc:`intro` section under "Related Projects", there are many other code bases you can hack on if you wish! + +Branching Strategy +------------------ + +Goals +~~~~~ + +The goals of the Dataverse branching strategy are: + +- allow for concurrent development +- only ship stable code + +We follow a simplified "git flow" model described at http://nvie.com/posts/a-successful-git-branching-model/ involving a "master" branch, a "develop" branch, and feature branches such as "1234-bug-fix". + +Branches +~~~~~~~~ + +The "master" Branch +******************* + +The "`master `_" branch represents released versions of Dataverse. As mentioned in the :doc:`making-releases` section, at release time we update the master branch to include all the code for that release. Commits are never made directly to master. Rather, master is updated only when we merge code into it from the "develop" branch. + +The "develop" Branch +******************** + +The "`develop `_" branch represents code that was stable enough to merge from a "feature" branch (described below) and that will make it into the next release. Like master, commits are never made to the develop branch. The develop branch is where integration occurs. Your goal is have your code merged into the develop branch after it has been reviewed. + +Feature Branches +**************** + +Feature branches are used for both developing features and fixing bugs. They are named after the GitHub issue they are meant to address, so create a GitHub issue if you need to. + +"3728-doc-apipolicy-fix" is an example of a fine name for your feature branch. It tells us that you are addressing https://github.com/IQSS/dataverse/issues/3728 and the "slug" is short, descriptive, and starts with the issue number. + +How to Make a Pull Request +-------------------------- + +Pull requests take all shapes and sizes, from a one-character typo fix to hundreds of files changing at once. Generally speaking, smaller pull requests are better so that they are easier to code review. That said, don't hold back on writing enough code or documentation to address the issue to the best of your ability. + +If you are writing code, please see :doc:`testing` for guidance on writing tests. + +The example of creating a pull request below has to do with fixing an important issue with the :doc:`documentation` but applies to fixing code as well. + +Find or Create a GitHub Issue +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For guidance on which issue to work on, please ask! Also, see https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md + +Let's say you want to tackle https://github.com/IQSS/dataverse/issues/3728 which points out a typo in a page of Dataverse's documentation. + +If you tell us your GitHub username we are happy to add you to the "read only" team at https://github.com/orgs/IQSS/teams/dataverse-readonly/members so that we can assign the issue to you while you're working on it. You can also tell us if you'd like to be added to the Dataverse Community Contributors spreadsheet at https://docs.google.com/spreadsheets/d/1o9DD-MQ0WkrYaEFTD5rF_NtyL8aUISgURsAXSL7Budk/edit?usp=sharing . + +Create a New Branch off the develop Branch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Always create your feature branch from the latest code in develop, pulling the latest code if necessary. As mentioned above, your branch should have a name like "3728-doc-apipolicy-fix" that starts with the issue number you are addressing, and ends with a short, descriptive name. + +Commit Your Change to Your New Branch +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Make your change in the form of a commit to that branch. Make sure the title of your commit includes a reference to the number of the issue it relates to, such as ``Fixed BlockedApiPolicy #3728``. Ideally the title is 50 characters or fewer, but don't worry about it. Use as much space in the body of the commit message as you need! + +Push Your Branch to GitHub +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Push your feature branch to your fork of Dataverse (or to the main repo at IQSS, if you have write access). Create a pull request based on the feature branch you pushed. As mentioned in https://github.com/IQSS/dataverse/blob/develop/CONTRIBUTING.md if you do not have access to advance your pull request into the "Code Review" column at https://waffle.io/IQSS/dataverse you should reach out to ask for it to be moved on your behalf. + +Make a Pull Request +~~~~~~~~~~~~~~~~~~~ + +Make a pull request to get approval to merge your changes into the develop branch. Feedback on the pull request template we use is welcome! The "connects to #3728" syntax is important because it's used at https://waffle.io/IQSS/dataverse to associate pull requests with issues. + +Here's an example of a pull request for issue #3728: https://github.com/IQSS/dataverse/pull/3827 + +Make Sure Your Pull Request Has Been Advanced to Code Review +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Now that you've made your pull request, your goal is to make sure it appears in the "Code Review" column at https://waffle.io/IQSS/dataverse + +Look at https://github.com/IQSS/dataverse/blob/master/CONTRIBUTING.md for various ways to reach out to developers who have enough access to the GitHub repo to move your issue and pull request to the "Code Review" column. + +How to Resolve Conflicts in Your Pull Request +--------------------------------------------- + +Unfortunately, pull requests can quickly become "stale" and unmergable as other pull requests are merged into the develop branch ahead of you. This is completely normal, and often occurs because other developers made their pull requests before you did. + +The Dataverse team may ping you to ask you to merge the latest from the develop branch into your branch and resolve merge conflicts. If this sounds daunting, please just say so and we will assist you. + +If you'd like to resolve the merge conflicts yourself, here are some steps to do so that make use of GitHub Desktop and Netbeans. + +**In GitHub Desktop:** + +1. Sync from develop. +2. Open the specific branch that's having the merge conflict. +3. Click "Update from develop". + +**In Netbeans:** + +4. Click Window -> Favorites and open your local Dataverse project folder in the Favorites panel. +5. In this file browser, you can follow the red cylinder icon to find files with merge conflicts. +6. Double click the red merge conflicted file. +7. Right click on the red tab for that file and select Git -> Resolve Conflicts. +8. Resolve on right or left (if you select "both" you can do finer edits after). +9. Save all changes + +**In GitHub Desktop:** + +10. Commit the merge (append issue number to end, e.g. #3728) and leave note about what was resolved. + +**In GitHub Issues:** + +11. Leave a comment for the Dataverse team that you have resolved the merge conflicts. + +---- + +Previous: :doc:`dev-environment` | Next: :doc:`testing` diff --git a/doc/sphinx-guides/source/index.rst b/doc/sphinx-guides/source/index.rst index 27db301c6e5..2ac49784659 100755 --- a/doc/sphinx-guides/source/index.rst +++ b/doc/sphinx-guides/source/index.rst @@ -3,10 +3,10 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Dataverse 4.6.2 Guides -====================== +Dataverse 4.7 Guides +==================== -These guides are for the most recent version of Dataverse. For the guides for **version 4.6.1** please go `here `_. +These guides are for the most recent version of Dataverse. For the guides for **version 4.6.2** please go `here `_. .. toctree:: :glob: diff --git a/doc/sphinx-guides/source/installation/administration.rst b/doc/sphinx-guides/source/installation/administration.rst index b08b510c434..ed1bb0fd775 100644 --- a/doc/sphinx-guides/source/installation/administration.rst +++ b/doc/sphinx-guides/source/installation/administration.rst @@ -3,7 +3,8 @@ Administration This section focuses on system and database administration tasks. Please see the :doc:`/user/index` for tasks having to do with having the "Admin" role on a dataverse or dataset. -.. contents:: :local: +.. contents:: |toctitle| + :local: Solr Search Index ----------------- @@ -18,15 +19,15 @@ There are two ways to perform a full reindex of the Dataverse search index. Star Clear and Reindex ~~~~~~~~~~~~~~~~~ -Clearing Data from Solr -....................... +**Clearing Data from Solr** +........................... Please note that the moment you issue this command, it will appear to end users looking at the home page that all data is gone! This is because the home page is powered by the search index. ``curl http://localhost:8080/api/admin/index/clear`` -Start Async Reindex -................... +**Start Async Reindex** +....................... Please note that this operation may take hours depending on the amount of data in your system. This known issue is being tracked at https://github.com/IQSS/dataverse/issues/50 @@ -37,13 +38,13 @@ Reindex in Place An alternative to completely clearing the search index is to reindex in place. -Clear Index Timestamps -...................... +**Clear Index Timestamps** +.......................... ``curl -X DELETE http://localhost:8080/api/admin/index/timestamps`` -Start or Continue Async Reindex -................................ +**Start or Continue Async Reindex** +................................... If indexing stops, this command should pick up where it left off based on which index timestamps have been set, which is why we start by clearing these timestamps above. These timestamps are stored in the ``dvobject`` database table. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index a8d8e875b5a..72504a18dca 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -8,7 +8,8 @@ Settings within Dataverse itself are managed via JVM options or by manipulating Once you have finished securing and configuring your Dataverse installation, proceed to the :doc:`administration` section. Advanced configuration topics are covered in the :doc:`r-rapache-tworavens`, :doc:`shibboleth` and :doc:`oauth2` sections. -.. contents:: :local: +.. contents:: |toctitle| + :local: Securing Your Installation -------------------------- @@ -115,6 +116,8 @@ Customizing the Root Dataverse As the person installing Dataverse you may or may not be a local metadata expert. You may want to have others sign up for accounts and grant them the "Admin" role at the root dataverse to configure metadata fields, templates, browse/search facets, guestbooks, etc. For more on these topics, consult the :doc:`/user/dataverse-management` section of the User Guide. +You are also welcome to adjust the theme of the root dataverse by adding a logo or changing header colors, etc. + Once this configuration is complete, your Dataverse installation should be ready for users to start playing with. That said, there are many more configuration options available, which will be explained below. Persistent Identifiers and Publishing Datasets @@ -219,6 +222,70 @@ Then run the create command: ``./asadmin $ASADMIN_OPTS create-jvm-options "\-Ddataverse.files.storage-driver-id=swift"`` +.. _Branding Your Installation: + +Branding Your Installation +-------------------------- + +The name of your root dataverse is the brand of your installation of Dataverse and appears in various places such as notifications. Notifications and support links also contain the name of your support team, which will either be "[YourBrand] Support" or a name of your choosing when you configure a name for the ``:SystemEmail`` database setting described below. + +Dataverse provides configurable options for easy-to-add (and maintain) custom branding for your Dataverse installation. Downloadable sample HTML and CSS files are provided below which you can edit as you see fit. It's up to you to create a directory in which to store these files, such as ``/var/www/dataverse`` in the examples below. + +You can add a custom welcome/homepage as well as other custom content, to further brand your installation and make it your own. Here are the custom branding and content options you can add: + +- Custom welcome/homepage +- Logo image to navbar +- Header +- Footer +- CSS stylesheet + +Custom Homepage +++++++++++++++++ + +Dataverse allows you to use a custom homepage or welcome page in place of the default root dataverse page. This allows for complete control over the look and feel of your installation's homepage. + +Download this sample: :download:`custom-homepage.html ` and place it at ``/var/www/dataverse/branding/custom-homepage.html``. Then run this curl command: + +``curl -X PUT -d '/var/www/dataverse/branding/custom-homepage.html' http://localhost:8080/api/admin/settings/:HomePageCustomizationFile`` + +Note that the ``custom-homepage.html`` file provided has a "Browse Data" button that assumes that your root dataverse still has an alias of "root". While you were branding your root dataverse, you may have changed the alias to "harvard" or "librascholar" or whatever and you should adjust the ``custom-homepage.html`` file as needed. + +For more background on what this curl command above is doing, see the "Database Settings" section below. If you decide you'd like to remove this setting, use the following curl command: + +``curl -X DELETE http://localhost:8080/api/admin/settings/:HomePageCustomizationFile`` + +Custom Navbar Logo ++++++++++++++++++++ + +Dataverse allows you to replace the default Dataverse icon and name branding in the navbar with your own custom logo. Note that this logo is separate from the *root dataverse theme* logo. + +Create a "navbar" folder in your Glassfish "logos" directory and place your custom logo there. By Glassfish default, it'll be located at ``/usr/local/glassfish4/glassfish/domains/domain1/docroot/logos/navbar/logo.png``. Then run this curl command: + +``curl -X PUT -d '/logos/navbar/logo.png' http://localhost:8080/api/admin/settings/:LogoCustomizationFile`` + +Note that the logo is expected to be small enough to fit comfortably in the navbar, perhaps only 30 pixels high. + +Custom Header ++++++++++++++ + +Download this sample: :download:`custom-header.html ` and place it at ``/var/www/dataverse/branding/custom-header.html``. Then run this curl command: + +``curl -X PUT -d '/var/www/dataverse/branding/custom-header.html' http://localhost:8080/api/admin/settings/:HeaderCustomizationFile`` + +Custom Footer ++++++++++++++ + +Download this sample: :download:`custom-footer.html ` and place it at ``/var/www/dataverse/branding/custom-footer.html``. Then run this curl command: + +``curl -X PUT -d '/var/www/dataverse/branding/custom-footer.html' http://localhost:8080/api/admin/settings/:FooterCustomizationFile`` + +Custom CSS Stylesheet ++++++++++++++++++++++ + +Download this sample: :download:`custom-stylesheet.css ` and place it at ``/var/www/dataverse/branding/custom-stylesheet.css``. Then run this curl command: + +``curl -X PUT -d '/var/www/dataverse/branding/custom-stylesheet.css' http://localhost:8080/api/admin/settings/:StyleCustomizationFile`` + Going Live: Launching Your Production Deployment ------------------------------------------------ @@ -391,10 +458,12 @@ This JVM setting is also part of **handles** configuration. The Handle.Net insta Database Settings ----------------- -These settings are stored in the ``setting`` table but can be read and modified via the "admin" endpoint of the :doc:`/api/native-api` for easy scripting. +These settings are stored in the ``setting`` database table but can be read and modified via the "admin" endpoint of the :doc:`/api/native-api` for easy scripting. The most commonly used configuration options are listed first. +The pattern you will observe in curl examples below is that an HTTP ``PUT`` is used to add or modify a setting. If you perform an HTTP ``GET`` (the default when using curl), the output will contain the value of the setting, if it has been set. You can also do a ``GET`` of all settings with ``curl http://localhost:8080/api/admin/settings`` which you may want to pretty-print by piping the output through a tool such as jq by appending ``| jq .``. If you want to remove a setting, use an HTTP ``DELETE`` such as ``curl -X DELETE http://localhost:8080/api/admin/settings/:GuidesBaseUrl`` . + :BlockedApiPolicy +++++++++++++++++ @@ -432,9 +501,31 @@ The key required to create users via API as documented at :doc:`/api/native-api` :SystemEmail ++++++++++++ -This is the email address that "system" emails are sent from such as password reset links. +This is the email address that "system" emails are sent from such as password reset links. Your Dataverse installation will not send mail without this setting in place. -``curl -X PUT -d "Support " http://localhost:8080/api/admin/settings/:SystemEmail`` +``curl -X PUT -d 'LibraScholar SWAT Team ' http://localhost:8080/api/admin/settings/:SystemEmail`` + +Note that only the email address is required, which you can supply without the ``<`` and ``>`` signs, but if you include the text, it's the way to customize the name of your support team, which appears in the "from" address in emails as well as in help text in the UI. + +:HomePageCustomizationFile +++++++++++++++++++++++++++ + +See :ref:`Branding Your Installation` above. + +:HeaderCustomizationFile +++++++++++++++++++++++++ + +See :ref:`Branding Your Installation` above. + +:FooterCustomizationFile +++++++++++++++++++++++++ + +See :ref:`Branding Your Installation` above. + +:StyleCustomizationFile ++++++++++++++++++++++++ + +See :ref:`Branding Your Installation` above. :FooterCopyright ++++++++++++++++ @@ -510,6 +601,7 @@ For systems using Postgresql 8.4 or older, the procedural language `plpgsql` sho We have provided an example :download:`here `. + :ApplicationTermsOfUse ++++++++++++++++++++++ @@ -540,6 +632,16 @@ Set ``:ExcludeEmailFromExport`` to prevent email addresses for dataset contacts ``curl -X PUT -d true http://localhost:8080/api/admin/settings/:ExcludeEmailFromExport`` +:NavbarAboutUrl ++++++++++++++++ + +Set ``NavbarAboutUrl`` to a fully-qualified url which will be used for the "About" link in the navbar. + +Note: The "About" link will not appear in the navbar until this option is set. + +``curl -X PUT -d http://dataverse.example.edu http://localhost:8080/api/admin/settings/:NavbarAboutUrl`` + + :GuidesBaseUrl ++++++++++++++ @@ -763,3 +865,29 @@ Set the base URL for the "Compute" button for a dataset. Set the base URL for the "Compute" button for a dataset. ``curl -X PUT -d 'Massachusetts Open Cloud (MOC)' http://localhost:8080/api/admin/settings/:CloudEnvironmentName`` + +:DataCaptureModuleUrl ++++++++++++++++++++++ + +The URL for your Data Capture Module (DCM) installation. This component is experimental and can be downloaded from https://github.com/sbgrid/data-capture-module . + +``curl -X PUT -d 'https://dcm.example.edu' http://localhost:8080/api/admin/settings/:DataCaptureModuleUrl`` + +:RepositoryStorageAbstractionLayerUrl ++++++++++++++++++++++++++++++++++++++ + +The URL for your Repository Storage Abstraction Layer (RSAL) installation. This component is experimental and can be downloaded from https://github.com/sbgrid/rsal . + +``curl -X PUT -d 'https://rsal.example.edu' http://localhost:8080/api/admin/settings/:RepositoryStorageAbstractionLayerUrl`` + +:UploadMethods +++++++++++++++ + +This setting is experimental and to be used with the Data Capture Module (DCM). For now, if you set the upload methods to ``dcm/rsync+ssh`` it will allow your users to download rsync scripts from the DCM. + +``curl -X PUT -d 'dcm/rsync+ssh' http://localhost:8080/api/admin/settings/:UploadMethods`` + +:DownloadMethods +++++++++++++++++ + +This setting is experimental and related to Repository Storage Abstraction Layer (RSAL). As of this writing it has no effect. diff --git a/doc/sphinx-guides/source/installation/data-capture-module.rst b/doc/sphinx-guides/source/installation/data-capture-module.rst new file mode 100644 index 00000000000..38039fdaf0f --- /dev/null +++ b/doc/sphinx-guides/source/installation/data-capture-module.rst @@ -0,0 +1,15 @@ +Data Capture Module +=================== + +Data Capture Module (DCM) is an experimental component that allows users to upload large datasets via rsync over ssh. Installation instructions can be found at https://github.com/sbgrid/data-capture-module . + +Once you have installed a DCM, you will need to configure two database settings on the Dataverse side. These settings are documented in the :doc:`config` section: + +- ``:DataCaptureModuleUrl`` should be set to the URL of a DCM you installed. +- ``:UploadMethods`` should be set to ``dcm/rsync+ssh``. + +This will allow your Dataverse installation to communicate with your DCM, so that Dataverse can download rsync scripts for your users. + +The rsync script can be downloaded from Dataverse via API using an authorized API token. In the curl example below, substitute ``{persistentId}`` with a DOI or Handle: + +``curl -H "X-Dataverse-key: $API_TOKEN" https://dataverse.example.edu/api/datasets/:persistentId/dataCaptureModule/rsync?persistentId={persistentId}`` diff --git a/doc/sphinx-guides/source/installation/geoconnect.rst b/doc/sphinx-guides/source/installation/geoconnect.rst index 5ac5f360f73..30d07f3f76c 100644 --- a/doc/sphinx-guides/source/installation/geoconnect.rst +++ b/doc/sphinx-guides/source/installation/geoconnect.rst @@ -1,6 +1,9 @@ Geoconnect ========== +.. contents:: |toctitle| + :local: + Geoconnect works as a middle layer, allowing geospatial data files in Dataverse to be visualized with Harvard WorldMap. To understand the feature from the user perspective, see the :doc:`/user/data-exploration/worldmap` section of the User Guide. diff --git a/doc/sphinx-guides/source/installation/index.rst b/doc/sphinx-guides/source/installation/index.rst index 469bb75a481..0223efcfaf2 100755 --- a/doc/sphinx-guides/source/installation/index.rst +++ b/doc/sphinx-guides/source/installation/index.rst @@ -6,7 +6,7 @@ Installation Guide ================== -Contents: +**Contents:** .. toctree:: @@ -21,3 +21,4 @@ Contents: geoconnect shibboleth oauth2 + data-capture-module diff --git a/doc/sphinx-guides/source/installation/installation-main.rst b/doc/sphinx-guides/source/installation/installation-main.rst index 44768212e34..a33d7698fab 100755 --- a/doc/sphinx-guides/source/installation/installation-main.rst +++ b/doc/sphinx-guides/source/installation/installation-main.rst @@ -4,7 +4,8 @@ Installation Now that the :doc:`prerequisites` are in place, we are ready to execute the Dataverse installation script (the "installer") and verify that the installation was successful by logging in with a "superuser" account. -.. contents:: :local: +.. contents:: |toctitle| + :local: .. _dataverse-installer: diff --git a/doc/sphinx-guides/source/installation/intro.rst b/doc/sphinx-guides/source/installation/intro.rst index 9a6b40cda89..a7eee7e963d 100644 --- a/doc/sphinx-guides/source/installation/intro.rst +++ b/doc/sphinx-guides/source/installation/intro.rst @@ -4,6 +4,9 @@ Introduction Welcome! Thanks for installing `Dataverse `_! +.. contents:: |toctitle| + :local: + Quick Links ----------- diff --git a/doc/sphinx-guides/source/installation/oauth2.rst b/doc/sphinx-guides/source/installation/oauth2.rst index 379bebc1daf..d2e18f8b89b 100644 --- a/doc/sphinx-guides/source/installation/oauth2.rst +++ b/doc/sphinx-guides/source/installation/oauth2.rst @@ -1,7 +1,8 @@ OAuth Login: ORCID, GitHub, Google ================================== -.. contents:: :local: +.. contents:: |toctitle| + :local: Introduction ------------ diff --git a/doc/sphinx-guides/source/installation/prep.rst b/doc/sphinx-guides/source/installation/prep.rst index 9452fdf5d3b..9662b5c40b6 100644 --- a/doc/sphinx-guides/source/installation/prep.rst +++ b/doc/sphinx-guides/source/installation/prep.rst @@ -8,7 +8,8 @@ Preparation We'll try to get you up and running as quickly as possible, but we thought you might like to hear about your options. :) -.. contents:: :local: +.. contents:: |toctitle| + :local: Choose Your Own Installation Adventure -------------------------------------- diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 0e909fb1570..a570e5ab84e 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -6,7 +6,8 @@ Before running the Dataverse installation script, you must install and configure You **may** find it helpful to look at how the configuration is done automatically by various tools such as Vagrant, Puppet, or Ansible. See the :doc:`prep` section for pointers on diving into these scripts. -.. contents:: :local: +.. contents:: |toctitle| + :local: Java ---- diff --git a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst index 8b6ca9cf2e2..4e394461e26 100644 --- a/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst +++ b/doc/sphinx-guides/source/installation/r-rapache-tworavens.rst @@ -3,8 +3,6 @@ TwoRavens ========= -.. contents:: :local: - TwoRavens is a web application for tabular data exploration and statistical analysis. It can be integrated with Dataverse, as an **optional** component. While TwoRavens was originally created at IQSS, its developers have since left the organization. Plans for the future of the Dataverse/TwoRavens collaboration are still being worked out. As @@ -39,8 +37,10 @@ rApache and a collection of required third-party R packages. The installation steps for these components are described in the individual sections of the document below. +.. contents:: |toctitle| + :local: -0. OVERVIEW +0. Overview +++++++++++ TwoRavens is itself a compact JavaScript application that **runs on the user's @@ -93,7 +93,7 @@ section ``1.b.`` where we explain how to completely erase all the previously built packages. -1. PREREQUISITES +1. Prerequisites ++++++++++++++++ a. httpd (Apache): diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst index 49ba9868f19..59562ba0696 100644 --- a/doc/sphinx-guides/source/installation/shibboleth.rst +++ b/doc/sphinx-guides/source/installation/shibboleth.rst @@ -1,8 +1,9 @@ Shibboleth ========== -.. contents:: :local: - +.. contents:: |toctitle| + :local: + Introduction ------------ diff --git a/doc/sphinx-guides/source/installation/upgrading.rst b/doc/sphinx-guides/source/installation/upgrading.rst index 4ad14bb67d6..66c491a3e87 100644 --- a/doc/sphinx-guides/source/installation/upgrading.rst +++ b/doc/sphinx-guides/source/installation/upgrading.rst @@ -2,6 +2,9 @@ Upgrading ========= +.. contents:: |toctitle| + :local: + When upgrading within Dataverse 4.x, you will need to follow the upgrade instructions for each intermediate version. Upgrades always involve deploying the latest war file but may also include running SQL scripts and updating the schema used by Solr. diff --git a/doc/sphinx-guides/source/style/foundations.rst b/doc/sphinx-guides/source/style/foundations.rst index 2899dfec20e..d90b617575f 100755 --- a/doc/sphinx-guides/source/style/foundations.rst +++ b/doc/sphinx-guides/source/style/foundations.rst @@ -3,6 +3,8 @@ Foundations Foundation elements are the very basic building blocks to create a page in Dataverse. Here we will outline how we've applied Bootstrap CSS to our UI, and how the CSS settings in our stylesheet mesh with it. Each section includes links to relevant parts of the official Bootstrap guides and other useful resources, where you can find more detailed documentation. We will also outline other UI resources like FontCustom and Socicon and how they are utilized. +.. contents:: |toctitle| + :local: Grid Layout =========== diff --git a/doc/sphinx-guides/source/style/index.rst b/doc/sphinx-guides/source/style/index.rst index 44ccc9ed2ab..ba6995e1b53 100755 --- a/doc/sphinx-guides/source/style/index.rst +++ b/doc/sphinx-guides/source/style/index.rst @@ -8,7 +8,7 @@ Style Guide This style guide is meant to help developers implement clear and appropriate UI elements consistent with the Dataverse Project's standards. -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/style/patterns.rst b/doc/sphinx-guides/source/style/patterns.rst index f51ecd35e35..80792bbdf68 100644 --- a/doc/sphinx-guides/source/style/patterns.rst +++ b/doc/sphinx-guides/source/style/patterns.rst @@ -3,6 +3,8 @@ Patterns Patterns are what emerge when using the foundation elements together with basic objects like buttons and alerts, more complex Javascript components from `Bootstrap `__ like tooltips and dropdowns, and AJAX components from `PrimeFaces `__ like datatables and commandlinks. +.. contents:: |toctitle| + :local: Navbar ====== @@ -27,19 +29,20 @@ When logged in, the account name is a dropdown menu, linking the user to account @@ -516,7 +519,7 @@ For our help/information, success, warning, and error message blocks we use a cu  Success! – The metadata for this dataset has been updated.
-  Error – The username, email address, or password you entered is invalid. Need assistance accessing your account? If you believe this is an error, please contact Dataverse Support for assistance. +  Error – The username, email address, or password you entered is invalid. Need assistance accessing your account? If you believe this is an error, please contact Root Support for assistance.
@@ -729,4 +732,4 @@ The modal is styled using the `Modal component `_. + +.. contents:: |toctitle| + :local: + Exploring and Analyzing Tabular files in Dataverse ================================================== @@ -36,7 +40,7 @@ The Summary tab shows summary statistics when a pebble is hovered over. If one r Modeling/Center Panel ===================== -The center panel displays a graphical representation of variable relations and denotes variables that have been tagged with certain properties. Variables may be tagged as either a dependent variable, a time series variable, or a cross sectional variable. Each of these are accomplished by clicking on the appropriate button at the top of the screen, and then clicking on a pebble in the center panel. You’ll notice that when a variable is tagged with a property, the fill color becomes white, and the outline (or stroke) of the pebble turns the color of property’s button. Note that to estimate a model, the dependent variable must be selected. +The center panel displays a graphical representation of variable relations and denotes variables that have been tagged with certain properties. Variables may be tagged as either a dependent variable, a time series variable, or a cross sectional variable. Each of these are accomplished by clicking on the appropriate button at the top of the screen, and then clicking on a pebble in the center panel. You’ll notice that when a variable is tagged with a property, the fill color becomes white, and the outline (or stroke) of the pebble turns the color of the property’s button. Note that to estimate a model, the dependent variable must be selected. Variable relations are specified by point-click-drag from one pebble to the other. When a path between pebbles has been specified, it is visually presented with a dotted arrow line and may be removed by pushing the delete key on your keyboard. @@ -77,4 +81,4 @@ Reset This is your start over button. Clicking this is equivalent to reloading the Web page or re-initiating TwoRavens. -Scenario Example: + diff --git a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst b/doc/sphinx-guides/source/user/data-exploration/worldmap.rst index b3a7ea7c815..fe6be851be8 100644 --- a/doc/sphinx-guides/source/user/data-exploration/worldmap.rst +++ b/doc/sphinx-guides/source/user/data-exploration/worldmap.rst @@ -3,7 +3,8 @@ WorldMap: Geospatial Data Exploration +++++++++++++++++++++++++++++++++++++ -.. contents:: :local: +.. contents:: |toctitle| + :local: Dataverse and WorldMap ====================== @@ -17,7 +18,7 @@ What is Geoconnect? Geoconnect is a platform that integrates Dataverse and WorldMap, allowing researchers to visualize their geospatial data. Geoconnect can be used to create maps of shapefiles or of tabular files containing geospatial information. Geoconnect is an optional component of Dataverse, so if you are interested in this feature but don't see it in the installation of Dataverse you are using, you should contact the support team for that installation and ask them to enable the Geoconnect feature. -If a data file's owner has created a map of that data using Geoconnect, you can view the map by clicking the "Explore" button. If the data is in the form of a shapefile, the button take you right to the map. If it's a tabular file, the Explore button will be a dropdown, and you'll need to select "Worldmap". +If a data file's owner has created a map of that data using Geoconnect, you can view the map by clicking the "Explore" button. If the data is in the form of a shapefile, the button takes you right to the map. If it's a tabular file, the Explore button will be a dropdown, and you'll need to select "Worldmap". Mapping shapefiles with Geoconnect ================================== diff --git a/doc/sphinx-guides/source/user/dataset-management.rst b/doc/sphinx-guides/source/user/dataset-management.rst index c921ab7523d..6be8981839e 100755 --- a/doc/sphinx-guides/source/user/dataset-management.rst +++ b/doc/sphinx-guides/source/user/dataset-management.rst @@ -5,16 +5,20 @@ A dataset in Dataverse is a container for your data, documentation, code, and th |image1| +.. contents:: |toctitle| + :local: + + Supported Metadata ================== A dataset contains three levels of metadata: #. **Citation Metadata**: any metadata that would be needed for generating a data citation and other general metadata that could be applied to any dataset; -#. **Domain specific Metadata**: with specific support currently for Social Science, Life Science, Geospatial, and Astronomy datasets; and +#. **Domain Specific Metadata**: with specific support currently for Social Science, Life Science, Geospatial, and Astronomy datasets; and #. **File-level Metadata**: varies depending on the type of data file - see *File Handling and Uploading* section below for more details). -For more details about what Citation and Domain specific metadata is supported please see our :ref:`user-appendix`. +For more details about what Citation and Domain Specific Metadata is supported please see our :ref:`user-appendix`. Note that once a dataset has been published its metadata may be exported. A button on the dataset page's metadata tab will allow a user to export the metadata of the most recently published version of the dataset. Currently supported export formats are DDI, Dublin Core and JSON. @@ -26,7 +30,7 @@ Adding a New Dataset #. To quickly get started, enter at minimum all the required fields with an asterisk (e.g., the Dataset Title, Author, Description, Contact Email and Subject) to get a Data Citation with a DOI. #. Scroll down to the "Files" section and click on "Select Files to Add" to add all the relevant files to your Dataset. - You can also upload your files directly from your Dropbox. **Tip:** You can drag and drop or select multiple files at a time from your desktop, + You can also upload your files directly from your Dropbox. **Tip:** You can drag and drop or select multiple files at a time from your desktop directly into the upload widget. Your files will appear below the "Select Files to Add" button where you can add a description and tags (via the "Edit Tag" button) for each file. Additionally, an MD5 checksum will be added for each file. If you upload a tabular file a :ref:`Universal Numerical Fingerprint (UNF) ` will be added to this file. #. Click the "Save Dataset" button when you are done. Your unpublished dataset is now created. @@ -45,19 +49,18 @@ File Handling + Uploading To upload new files to a dataset, click the "Edit" button at the top of the dataset page and from the dropdown list select "Files (Upload)" or click the "Upload Files" button above the files table in the Files tab. From either option you will be brought to the Upload Files page for that dataset. -Once you have uploaded files, you will be able to edit the file metadata, restrict, add tags. Click "Save Changes" to complete the upload. If you uploaded a file by mistake, you can delete them before saving by clicking the checkbox to select the file, and then clicking the "Delete" button above the files table. +Once you have uploaded files, you will be able to edit file metadata, restrict access to files, and/or add tags. Click "Save Changes" to complete the upload. If you uploaded a file by mistake, you can delete it before saving by clicking the checkbox to select the file, and then clicking the "Delete" button above the Files Table. File upload limit size varies based on Dataverse installation. The file upload size limit can be found in the text above where files are uploaded in the application. If you have further questions, contact support for that installation by clicking on the Support link at the top of the application. -The file types listed below are supported by additional functionality, which can include downloading in different formats, subsets, file-level metadata preservation, file-level data citation; and exploration -through data visualization and analysis. +The file types listed below are supported by additional functionality, which can include downloading in different formats, subsets, file-level metadata preservation, file-level data citation; and exploration through data visualization and analysis. Tabular Data Files ------------------ Files in certain formats - Stata, SPSS, R, Excel(xlsx) and CSV - may be ingested as tabular data (see "Tabular Data Ingest" section for details). Tabular data files can be further explored and manipulated with `TwoRavens <../user/data-exploration/tworavens.html>`_ - a statistical data exploration application integrated with Dataverse. It allows the user to run statistical models, view summary statistics, download subsets of variable vectors and more. To start, click on the "Explore" button, found next to each relevant tabular file (the application will be opened in a new window). To download subsets of variables click on the "Download" button found next to a relevant tabular file and select "Data Subset" in the dropdown menu. You will then be able to create your subset using the interface opened in a new window (this functionality is also provided by the `TwoRavens <../user/data-exploration/tworavens.html>`_ project). See the `TwoRavens documentation section <../user/data-exploration/tworavens.html>`_ for more information. -For example, the ingest functionality for tabular files in the Harvard Dataverse, a file can only be up to 2GB in size. To use the ingest functionality for RData files, a file can only be up to 1MB in size, however, to upload a RData file without using ingest, a file can be up to 2GB in size. +For example, for the ingest functionality for tabular files in Harvard Dataverse, a file can only be up to 2GB in size. To use the ingest functionality for RData files, a file can only be up to 1MB in size. However, to upload a RData file without using ingest, a file can be up to 2GB in size. Additional download options available for tabular data (found in the same drop-down menu under the "Download" button): @@ -143,7 +146,7 @@ Edit Files Edit File Metadata ------------------ -Go to the dataset you would like to edit where you will see the listing of files. Select the files you would like to edit by using either the Select All checkbox or individually selecting files. Next, click the "Edit Files" button above the file table and from the dropdown menu select if you would like to: +Go to the dataset you would like to edit, where you will see the listing of files. Select the files you would like to edit by using either the Select All checkbox or individually selecting files. Next, click the "Edit Files" button above the file table and from the dropdown menu select if you would like to: - Delete the selected files - Edit the file metadata (file name, description) for the selected files @@ -174,7 +177,7 @@ After successfully replacing a file, a new dataset draft version will be created Terms ===== -In the Terms tab, which can also be found by clicking on the Edit dropdown button of a Dataset, you can setup how users can use your data once they have downloaded it (CC0 waiver or custom Terms of Use), how they can access your data if you have files that are restricted (terms of access), and enable a Guestbook for your dataset so that you can track who is using your data and for what purposes. These are explained in further detail below: +In the Terms tab, which can also be found by clicking on the Edit dropdown button of a Dataset, you can set up how users can use your data once they have downloaded it (CC0 waiver or custom Terms of Use), how they can access your data if you have files that are restricted (terms of access), and enable a Guestbook for your dataset so that you can track who is using your data and for what purposes. These are explained in further detail below: CC0 Waiver + Dataset Terms of Use --------------------------------- @@ -216,7 +219,7 @@ Dataset permissions can be found from the dataset by clicking the "Edit" button The dataset permissions page has two sections: Users/Groups and Roles. In the top, "Users/Groups" panel, you can find all the users and groups that have access to your dataset. To give someone access to view your unpublished dataset or edit your published or unpublished dataset, click on the "Assign Roles to Users/Groups" button in the Users/Groups section. -The panel below that is "Roles" where you can find all the roles set up in your dataverse, that you can assign to users and groups. These roles are set at your dataverse-level and are displayed here at the dataset-level as a reference for when you are granting permission to users and/or groups. +The panel below that is "Roles", where you can find all the roles set up in your dataverse, which you can assign to users and groups. These roles are set at your dataverse-level and are displayed here at the dataset-level as a reference for when you are granting permission to users and/or groups. File-Level ---------- @@ -328,9 +331,9 @@ There is also a Versions tab on the file page. The versions table for a file dis Version Details --------------- -To view what has exactly changed starting from the originally published version to any subsequent published versions: click the Versions tab on the dataset page to see all versions and changes made for that particular dataset. +To view exactly what has changed, starting from the originally published version to any subsequent published versions: click the Versions tab on the dataset page to see all versions and changes made for that particular dataset. -Once you have more than one version (can be version 1 and a draft), you can click the "View Details" link next to each summary to learn more about the metadata fields and files that were either added or edited. You can also click the checkboxes to select any two dataset versions, then click the "View Differences" button to open the Version Differences Details popup and compare the differences between them. +Once you have more than one version (this can simply be version 1 and a draft), you can click the "View Details" link next to each summary to learn more about the metadata fields and files that were either added or edited. You can also click the checkboxes to select any two dataset versions, then click the "View Differences" button to open the Version Differences Details popup and compare the differences between them. .. _deaccession: diff --git a/doc/sphinx-guides/source/user/dataverse-management.rst b/doc/sphinx-guides/source/user/dataverse-management.rst index b3a1d7e95f1..82755773b90 100755 --- a/doc/sphinx-guides/source/user/dataverse-management.rst +++ b/doc/sphinx-guides/source/user/dataverse-management.rst @@ -9,6 +9,9 @@ Once a user creates a dataverse they, by default, become the administrator of that dataverse. The dataverse administrator has access to manage the settings described in this guide. +.. contents:: |toctitle| + :local: + Create a Dataverse (Within the "Root" Dataverse) =================================================== @@ -22,7 +25,7 @@ Creating a dataverse is easy but first you must be a registered user (see :doc:` * **Affiliation**: Add any Affiliation that can be associated to this particular dataverse (e.g., project name, institute name, department name, journal name, etc). This is automatically filled out if you have added an affiliation for your user account. * **Description**: Provide a description of this dataverse. This will display on the home page of your dataverse and in the search result list. The description field supports certain HTML tags (, ,
,
, , ,
,
,
, ,
,

-

, , , ,
  • ,
      ,

      ,

      , , , , , , 
        ). * **Category**: Select a category that best describes the type of dataverse this will be. For example, if this is a dataverse for an individual researcher's datasets, select Researcher. If this is a dataverse for an institution, select Organization & Institution. - * **Choose the sets of Metadata Elements for datasets in this dataverse**: by default the metadata elements will be from the host dataverse that this new dataverse is created in. Dataverse offers metadata standards for multiple domains. To learn more about the metadata standards in Dataverse please check out the :doc:`/user/appendix`. + * **Choose the sets of Metadata Elements for datasets in this dataverse**: By default the metadata elements will be from the host dataverse that this new dataverse is created in. Dataverse offers metadata standards for multiple domains. To learn more about the metadata standards in Dataverse please check out the :doc:`/user/appendix`. * **Select facets for this dataverse**: by default the facets that will appear on your dataverse landing page will be from the host dataverse that this new dataverse was created in. The facets are simply metadata fields that can be used to help others easily find dataverses and datasets within this dataverse. You can select as many facets as you would like. #. Selected metadata elements are also used to pick which metadata fields you would like to use for creating templates for your datasets. Metadata fields can be hidden, or selected as required or optional. Once you have selected all the fields you would like to use, you can create your template(s) after you finish creating your dataverse. #. Click "Create Dataverse" button and you're done! @@ -51,7 +54,7 @@ General Information The General Information page is how you edit the information you filled in while creating your dataverse. If you need to change or add a contact email address, this is the place to do it. Additionally, you can update the metadata elements used for datasets within the dataverse, change which metadata fields are hidden, required, or optional, and update the facets you would like displayed for browsing the dataverse. If you plan on using templates, you need to select the metadata fields on the General Information page. -Tip: The metadata fields you select as required, will appear on the Create Dataset form when someone goes to add a dataset to the dataverse. +Tip: The metadata fields you select as required will appear on the Create Dataset form when someone goes to add a dataset to the dataverse. .. _theme: @@ -115,7 +118,7 @@ You can also give access to a Dataverse user to allow them to access an unpublis Dataset Templates ====================== -Templates are useful when you have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in or want to use a custom set of Terms of Use and Access for multiple datasets in a dataverse. In Dataverse 4.0, templates are created at the dataverse level, can be deleted (so it does not show for future datasets), set to default (not required), or can be copied so you do not have to start over when creating a new template with similiar metadata from another template. When a template is deleted, it does not impact the datasets that have used the template already. +Templates are useful when you have several datasets that have the same information in multiple metadata fields that you would prefer not to have to keep manually typing in, or if you want to use a custom set of Terms of Use and Access for multiple datasets in a dataverse. In Dataverse 4.0, templates are created at the dataverse level, can be deleted (so it does not show for future datasets), set to default (not required), or can be copied so you do not have to start over when creating a new template with similiar metadata from another template. When a template is deleted, it does not impact the datasets that have used the template already. How do you create a template? diff --git a/doc/sphinx-guides/source/user/find-use-data.rst b/doc/sphinx-guides/source/user/find-use-data.rst index 5395593a2ad..f62877a32fc 100755 --- a/doc/sphinx-guides/source/user/find-use-data.rst +++ b/doc/sphinx-guides/source/user/find-use-data.rst @@ -1,6 +1,9 @@ Finding and Using Data +++++++++++++++++++++++ +.. contents:: |toctitle| + :local: + Finding Data ============= diff --git a/doc/sphinx-guides/source/user/index.rst b/doc/sphinx-guides/source/user/index.rst index 9d231cb5f6d..05973e6fac9 100755 --- a/doc/sphinx-guides/source/user/index.rst +++ b/doc/sphinx-guides/source/user/index.rst @@ -6,7 +6,7 @@ User Guide ================================================= -Contents: +**Contents:** .. toctree:: diff --git a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst index 87369471567..f716a6d4428 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/csv.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/csv.rst @@ -1,6 +1,9 @@ CSV ++++++ +.. contents:: |toctitle| + :local: + Ingest of Comma-Separated Values files as tabular data. ------------------------------------------------------- @@ -9,7 +12,7 @@ Dataverse will make an attempt to turn CSV files uploaded by the user into tabul Main formatting requirements: ----------------------------- -The first line must contan a comma-separated list of the variable names; +The first line must contain a comma-separated list of the variable names; All the lines that follow must contain the same number of comma-separated values as the first, variable name line. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst index ea51cda1870..b8612092f06 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/excel.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/excel.rst @@ -1,6 +1,9 @@ Excel +++++++ +.. contents:: |toctitle| + :local: + Excel files (**New** in Dataverse 4.0!) Note: only the "new", XLSX Excel files are supported. We are not planning to add support for the old-style, binary XLS files. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst index 12ae430bd67..4f71dd9944f 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/ingestprocess.rst @@ -6,6 +6,8 @@ the application and what happens during the ingest process, as the files uploaded by the user are processed and converted into the archival format in the Dataverse application. +.. contents:: |toctitle| + :local: What Happens During this "Ingest"? =================================== diff --git a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst index ae2cc6cf7fe..0707051b5a2 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/rdata.rst @@ -3,6 +3,8 @@ R Data Format Support for R (.RData) files has been introduced in DVN 3.5. +.. contents:: |toctitle| + :local: Overview. =========== diff --git a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst index 86a7d864851..71bc61265c3 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/spss.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/spss.rst @@ -3,6 +3,9 @@ SPSS SPSS data files (POR and SAV formats). +.. contents:: |toctitle| + :local: + Supported Versions ------------------ diff --git a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst index 764bc815a2f..e17e7321eae 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/stata.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/stata.rst @@ -1,6 +1,9 @@ Stata ++++++++ +.. contents:: |toctitle| + :local: + Of all the third party statistical software providers, Stata does the best job at documenting the internal format of their files, by far. And at making that documentation freely and easily available to developers (yes, we are looking at you, SPSS). Because of that, Stata is the best supported format for tabular data ingest. diff --git a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst index b5104a5a2c0..6078d37ee84 100644 --- a/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst +++ b/doc/sphinx-guides/source/user/tabulardataingest/supportedformats.rst @@ -1,8 +1,11 @@ Supported File Formats +++++++++++++++++++++++++++++ +.. contents:: |toctitle| + :local: + Tabular Data ingest supports the following file formats: -(see the sections below for more information on each of the supported formats) + ================================ ================================== File format Versions supported @@ -13,3 +16,5 @@ R up to 3 Excel XLSX only (XLS is NOT supported) CSV (comma-separated values) (limited support) ================================ ================================== + +See the subsections in the left sidebar for more information on each of these supported formats. \ No newline at end of file diff --git a/pom.xml b/pom.xml index 98ea5a047ad..ba4886dda60 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ edu.harvard.iq dataverse - 4.6.2 + 4.7 war dataverse diff --git a/scripts/api/setup-optional-harvard.sh b/scripts/api/setup-optional-harvard.sh index 4bcae4c8685..c1815998e6e 100755 --- a/scripts/api/setup-optional-harvard.sh +++ b/scripts/api/setup-optional-harvard.sh @@ -21,7 +21,7 @@ echo "- Enabling Geoconnect" curl -s -X PUT -d true "$SERVER/admin/settings/:GeoconnectCreateEditMaps" curl -s -X PUT -d true "$SERVER/admin/settings/:GeoconnectViewMaps" echo "- Setting system email" -curl -X PUT -d "Dataverse Support " http://localhost:8080/api/admin/settings/:SystemEmail +curl -X PUT -d "Harvard Dataverse Support " http://localhost:8080/api/admin/settings/:SystemEmail curl -X PUT -d ", The President & Fellows of Harvard College" http://localhost:8080/api/admin/settings/:FooterCopyright echo "- Setting up the Harvard Shibboleth institutional group" curl -s -X POST -H 'Content-type:application/json' --upload-file data/shibGroupHarvard.json "$SERVER/admin/groups/shib?key=$adminKey" diff --git a/scripts/database/upgrades/upgrade_v4.6.2_to_v4.7.sql b/scripts/database/upgrades/upgrade_v4.6.2_to_v4.7.sql new file mode 100644 index 00000000000..901b013a104 --- /dev/null +++ b/scripts/database/upgrades/upgrade_v4.6.2_to_v4.7.sql @@ -0,0 +1,2 @@ +--Uncomment to preserve "Dataverse" at end of each dataverse name. +--UPDATE dataverse SET name = name || ' Dataverse'; diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index 5c3fd33a754..ab8e29b663b 100755 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -90,11 +90,6 @@ body.skip=Skip to main content # dataverse_footer.xhtml -footer.codeAvailable=Code available at -footer.dataverseOnGitHub=Dataverse On GitHub -footer.dataverseProjectOn=Dataverse Project on -footer.Twitter=Twitter -footer.dataScienceIQSS=Developed at the Institute for Quantitative Social Science footer.copyright=Copyright © {0} footer.widget.datastored=Data is stored at {0}. footer.widget.login=Log in to @@ -112,11 +107,11 @@ messages.validation.msg=Required fields were missed or there was a validation er # contactFormFragment.xhtml -contact.header=Contact Dataverse Support +contact.header=Contact {0} contact.dataverse.header=Email Dataverse Contact contact.dataset.header=Email Dataset Contact contact.to=To -contact.support=Dataverse Support +contact.support=Support contact.from=From contact.from.required=User email is required. contact.from.invalid=Email is invalid. @@ -154,7 +149,7 @@ wasReturnedByReviewer=, was returned by the curator of toReview=Don't forget to publish it or send it back to the contributor! worldMap.added=dataset had a WorldMap layer data added to it. # Bundle file editors, please note that "notification.welcome" is used in a unit test. -notification.welcome=Welcome to {0} Dataverse! Get started by adding or finding data. Have questions? Check out the {1}. Want to test out Dataverse features? Use our {2}. Also, check for your welcome email to verify your address. +notification.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the {1}. Want to test out Dataverse features? Use our {2}. Also, check for your welcome email to verify your address. notification.demoSite=Demo Site notification.requestFileAccess=File access requested for dataset: {0}. notification.grantFileAccess=Access granted for files in dataset: {0}. @@ -498,38 +493,38 @@ harvestserver.viewEditDialog.btn.save=Save Changes #MailServiceBean.java -notification.email.create.dataverse.subject=Dataverse: Your dataverse has been created -notification.email.create.dataset.subject=Dataverse: Your dataset has been created -notification.email.request.file.access.subject=Dataverse: Access has been requested for a restricted file -notification.email.grant.file.access.subject=Dataverse: You have been granted access to restricted file -notification.email.rejected.file.access.subject=Dataverse: Your request for access to restricted file has been rejected -notification.email.update.maplayer=Dataverse: WorldMap layer added to dataset -notification.email.maplayer.deletefailed.subject=Failed to delete WorldMap layer +notification.email.create.dataverse.subject={0}: Your dataverse has been created +notification.email.create.dataset.subject={0}: Your dataset has been created +notification.email.request.file.access.subject={0}: Access has been requested for a restricted file +notification.email.grant.file.access.subject={0}: You have been granted access to a restricted file +notification.email.rejected.file.access.subject={0}: Your request for access to a restricted file has been rejected +notification.email.update.maplayer={0}: WorldMap layer added to dataset +notification.email.maplayer.deletefailed.subject={0}: Failed to delete WorldMap layer notification.email.maplayer.deletefailed.text=We failed to delete the WorldMap layer associated with the restricted file {0}, and any related data that may still be publicly available on the WorldMap site. Please try again, or contact WorldMap and/or Dataverse support. (Dataset: {1}) -notification.email.submit.dataset.subject=Dataverse: Your dataset has been submitted for review -notification.email.publish.dataset.subject=Dataverse: Your dataset has been published -notification.email.returned.dataset.subject=Dataverse: Your dataset has been returned -notification.email.create.account.subject=Dataverse: Your account has been created -notification.email.assign.role.subject=Dataverse: You have been assigned a role -notification.email.revoke.role.subject=Dataverse: Your role has been revoked -notification.email.verifyEmail.subject=Dataverse: Verify your email address +notification.email.submit.dataset.subject={0}: Your dataset has been submitted for review +notification.email.publish.dataset.subject={0}: Your dataset has been published +notification.email.returned.dataset.subject={0}: Your dataset has been returned +notification.email.create.account.subject={0}: Your account has been created +notification.email.assign.role.subject={0}: You have been assigned a role +notification.email.revoke.role.subject={0}: Your role has been revoked +notification.email.verifyEmail.subject={0}: Verify your email address notification.email.greeting=Hello, \n # Bundle file editors, please note that "notification.email.welcome" is used in a unit test -notification.email.welcome=Welcome to Dataverse! Get started by adding or finding data. Have questions? Check out the User Guide at {0}/{1}/user/ or contact Dataverse Support for assistance. Want to test out Dataverse features? Use our Demo Site at https://demo.dataverse.org +notification.email.welcome=Welcome to {0}! Get started by adding or finding data. Have questions? Check out the User Guide at {1}/{2}/user or contact {3} at {4} for assistance. notification.email.welcomeConfirmEmailAddOn=\n\nPlease verify your email address at {0}. Note, the verify link will expire after {1}. Send another verification email by visiting your account page. notification.email.requestFileAccess=File access requested for dataset: {0}. Manage permissions at {1}. notification.email.grantFileAccess=Access granted for files in dataset: {0} (view at {1}). notification.email.rejectFileAccess=Access rejected for requested files in dataset: {0} (view at {1}). # Bundle file editors, please note that "notification.email.createDataverse" is used in a unit test -notification.email.createDataverse=Your new dataverse named {0} (view at {1} ) was created in {2} (view at {3} ). To learn more about what you can do with your dataverse, check out the Dataverse Management - Dataverse User Guide at {4}/{5}/user/dataverse-management.html . +notification.email.createDataverse=Your new dataverse named {0} (view at {1} ) was created in {2} (view at {3} ). To learn more about what you can do with your dataverse, check out the Dataverse Management - User Guide at {4}/{5}/user/dataverse-management.html . # Bundle file editors, please note that "notification.email.createDataset" is used in a unit test -notification.email.createDataset=Your new dataset named {0} (view at {1} ) was created in {2} (view at {3} ). To learn more about what you can do with a dataset, check out the Dataset Management - Dataset User Guide at {4}/{5}/user/dataset-management.html . -notification.email.wasSubmittedForReview={0} (view at {1}) was submitted for review to be published in {2} (view at {3}). Don't forget to publish it or send it back to the contributor\! +notification.email.createDataset=Your new dataset named {0} (view at {1} ) was created in {2} (view at {3} ). To learn more about what you can do with a dataset, check out the Dataset Management - User Guide at {4}/{5}/user/dataset-management.html . +notification.email.wasSubmittedForReview={0} (view at {1}) was submitted for review to be published in {2} (view at {3}). Don't forget to publish it or send it back to the contributor\! notification.email.wasReturnedByReviewer={0} (view at {1}) was returned by the curator of {2} (view at {3}). notification.email.wasPublished={0} (view at {1}) was published in {2} (view at {3}). notification.email.worldMap.added={0} (view at {1}) had WorldMap layer data added to it. -notification.email.closing=\n\nThank you,\nThe Dataverse Project +notification.email.closing=\n\nThank you,\n{0} notification.email.assignRole=You are now {0} for the {1} "{2}" (view at {3}). notification.email.revokeRole=One of your roles for the {0} "{1}" has been revoked (view at {2}). notification.email.changeEmail=Hello, {0}.{1}\n\nPlease contact us if you did not intend this change or if you need assistance. @@ -537,9 +532,9 @@ hours=hours hour=hour minutes=minutes minute=minute -notification.email.checksumfail.subject=Dataverse: Your upload failed checksum validation. -notification.email.import.filesystem.subject=Dataverse: Your file import job has completed -notification.email.import.checksum.subject=Dataverse: Your file checksum job has completed +notification.email.checksumfail.subject={0}: Your upload failed checksum validation +notification.email.import.filesystem.subject={0}: Your file import job has completed +notification.email.import.checksum.subject={0}: Your file checksum job has completed # passwordreset.xhtml @@ -561,7 +556,7 @@ passwdReset.resetBtn=Reset Password # dataverse.xhtml -dataverse.title=The project, department, university, or professor this dataverse will contain data for. +dataverse.title=The project, department, university, professor, or journal this dataverse will contain data for. dataverse.enterName=Enter name... dataverse.host.title=The dataverse which contains this data. dataverse.identifier.title=Short name used for the URL of this dataverse. @@ -666,7 +661,7 @@ dataverse.update.failure=This dataverse was not able to be updated. # advanced.xhtml advanced.search.header.dataverses=Dataverses -advanced.search.dataverses.name.tip=The project, department, university, or professor this Dataverse will contain data for. +advanced.search.dataverses.name.tip=The project, department, university, professor, or journal this Dataverse will contain data for. advanced.search.dataverses.affiliation.tip=The organization with which this Dataverse is affiliated. advanced.search.dataverses.description.tip=A summary describing the purpose, nature, or scope of this Dataverse. advanced.search.dataverses.subject.tip=Domain-specific Subject Categories that are topically relevant to this Dataverse. @@ -951,7 +946,7 @@ dataset.manageTemplates.tab.action.btn.delete.dialog.header=Delete Template dataset.manageTemplates.tab.action.btn.view.dialog.header=Dataset Template Preview dataset.manageTemplates.tab.action.btn.view.dialog.datasetTemplate=Dataset Template dataset.manageTemplates.tab.action.btn.view.dialog.datasetTemplate.title=The dataset template which prepopulates info into the form automatically. -dataset.manageTemplates.tab.action.noedit.createdin=Template created at {0} Dataverse +dataset.manageTemplates.tab.action.noedit.createdin=Template created at {0} dataset.manageTemplates.delete.usedAsDefault=This template is the default template for the following dataverse(s). It will be removed as default as well. dataset.manageTemplates.info.message.notEmptyTable=Create, clone, edit, view, or delete dataset templates. Create a dataset template to prefill metadata fields with standard values, such as author affiliation, to help users create datasets in this dataverse. You can also add help text directly into the metadata fields to give users more information on what to add to these metadata fields. @@ -1034,7 +1029,7 @@ dataset.manageGuestbooks.tab.action.btn.view.dialog.datasetGuestbook=Guestbook N dataset.manageGuestbooks.tab.action.btn.viewCollectedData.dialog.header=Dataset Guestbook Collected Data dataset.manageGuestbooks.tab.action.btn.view.dialog.userCollectedData.title=User data collected by the guestbook. dataset.manageGuestbooks.tab.action.btn.view.dialog.userCollectedData=Collected Data -dataset.manageGuestbooks.tab.action.noedit.createdin=Guestbook created at {0} Dataverse +dataset.manageGuestbooks.tab.action.noedit.createdin=Guestbook created at {0} dataset.manageGuestbooks.message.deleteSuccess=The guestbook has been deleted. dataset.manageGuestbooks.message.deleteFailure=The guestbook cannot be deleted. dataset.manageGuestbooks.message.editSuccess=The guestbook has been updated. @@ -1383,11 +1378,11 @@ file.dataFilesTab.terms.list.termsOfAccess.addInfo.studyCompletion.title=Relatio file.dataFilesTab.terms.list.guestbook=Guestbook file.dataFilesTab.terms.list.guestbook.title=User information (i.e., name, email, institution, and position) will be collected when files are downloaded. file.dataFilesTab.terms.list.guestbook.noSelected.tip=No guestbook is assigned to this dataset, you will not be prompted to provide any information on file download. -file.dataFilesTab.terms.list.guestbook.noSelected.admin.tip=There are no guestbooks available in {0} Dataverse to assign to this dataset. +file.dataFilesTab.terms.list.guestbook.noSelected.admin.tip=There are no guestbooks available in {0} to assign to this dataset. file.dataFilesTab.terms.list.guestbook.inUse.tip=The following guestbook will prompt a user to provide additional information when downloading a file. file.dataFilesTab.terms.list.guestbook.viewBtn=Preview Guestbook file.dataFilesTab.terms.list.guestbook.select.tip=Select a guestbook to have a user provide additional information when downloading a file. -file.dataFilesTab.terms.list.guestbook.noAvailable.tip=There are no guestbooks enabled in {0} Dataverse. To create a guestbook, return to {0} Dataverse, click the "Edit" button and select the "Dataset Guestbooks" option. +file.dataFilesTab.terms.list.guestbook.noAvailable.tip=There are no guestbooks enabled in {0}. To create a guestbook, return to {0}, click the "Edit" button and select the "Dataset Guestbooks" option. file.dataFilesTab.terms.list.guestbook.clearBtn=Clear Selection file.dataFilesTab.versions=Versions @@ -1460,6 +1455,10 @@ file.viewDiffDialog.description=Description file.viewDiffDialog.fileReplaced=File Replaced file.viewDiffDialog.filesReplaced=File(s) Replaced file.viewDiffDialog.files.header=Files +file.viewDiffDialog.msg.draftFound= This is the "DRAFT" version. +file.viewDiffDialog.msg.draftNotFound=The "DRAFT" version was not found. +file.viewDiffDialog.msg.versionFound= This is version "{0}". +file.viewDiffDialog.msg.versionNotFound=Version "{0}" was not found. file.metadataTip=Metadata Tip: After adding the dataset, click the Edit Dataset button to add more metadata. file.addBtn=Save Dataset file.dataset.allFiles=All Files from this Dataset diff --git a/src/main/java/edu/harvard/iq/dataverse/AbstractIdServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/AbstractIdServiceBean.java index 9142893f7cb..1ee5fabbbcc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/AbstractIdServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/AbstractIdServiceBean.java @@ -53,7 +53,7 @@ protected HashMap addBasicMetadata(Dataset datasetIn, HashMapGET and POST + * methods. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + protected void processRequest(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + response.setContentType("text/html;charset=UTF-8"); + + String customFileType = request.getParameter("customFileType"); + String filePath = getFilePath(customFileType); + + Path physicalPath = Paths.get(filePath); + try { + File fileIn = physicalPath.toFile(); + if (fileIn != null) { + FileInputStream inputStream = new FileInputStream(fileIn); + + BufferedReader in = new BufferedReader(new InputStreamReader(inputStream)); + String line; + + StringBuilder responseData = new StringBuilder(); + try (PrintWriter out = response.getWriter()) { + + while ((line = in.readLine()) != null) { + responseData.append(line); + out.println(line); + } + } + + inputStream.close(); + + + } else { + /* + If the file doesn't exist or it is unreadable we don't care + */ + } + + } catch (Exception e) { + /* + If the file doesn't exist or it is unreadable we don't care + */ + } + + } + + private String getFilePath(String fileTypeParam){ + + String nonNullDefaultIfKeyNotFound = ""; + + if (fileTypeParam.equals(CustomizationConstants.fileTypeHomePage)) { + + // Homepage + return settingsService.getValueForKey(SettingsServiceBean.Key.HomePageCustomizationFile, nonNullDefaultIfKeyNotFound); + + } else if (fileTypeParam.equals(CustomizationConstants.fileTypeHeader)) { + + // Header + return settingsService.getValueForKey(SettingsServiceBean.Key.HeaderCustomizationFile, nonNullDefaultIfKeyNotFound); + + } else if (fileTypeParam.equals(CustomizationConstants.fileTypeFooter)) { + + // Footer + return settingsService.getValueForKey(SettingsServiceBean.Key.FooterCustomizationFile, nonNullDefaultIfKeyNotFound); + + } else if (fileTypeParam.equals(CustomizationConstants.fileTypeStyle)) { + + // Style (css) + return settingsService.getValueForKey(SettingsServiceBean.Key.StyleCustomizationFile, nonNullDefaultIfKeyNotFound); + + } else if (fileTypeParam.equals(CustomizationConstants.fileTypeLogo)) { + + // Logo for installation - appears in header + return settingsService.getValueForKey(SettingsServiceBean.Key.LogoCustomizationFile, nonNullDefaultIfKeyNotFound); + } + + + return ""; + } + + // + /** + * Handles the HTTP GET method. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + processRequest(request, response); + } + + /** + * Handles the HTTP POST method. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException if a servlet-specific error occurs + * @throws IOException if an I/O error occurs + */ + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + processRequest(request, response); + } + + /** + * Returns a short description of the servlet. + * + * @return a String containing servlet description + */ + @Override + public String getServletInfo() { + return "Short description"; + }// + +} \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index f1d25759d69..a173e9fdc33 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -160,12 +160,18 @@ public enum DisplayMode { DataverseRequestServiceBean dvRequestService; @Inject DatasetVersionUI datasetVersionUI; - @Inject PermissionsWrapper permissionsWrapper; - @Inject FileDownloadHelper fileDownloadHelper; - @Inject TwoRavensHelper twoRavensHelper; - @Inject WorldMapPermissionHelper worldMapPermissionHelper; - @Inject ThumbnailServiceWrapper thumbnailServiceWrapper; - + @Inject + PermissionsWrapper permissionsWrapper; + @Inject + FileDownloadHelper fileDownloadHelper; + @Inject + TwoRavensHelper twoRavensHelper; + @Inject + WorldMapPermissionHelper worldMapPermissionHelper; + @Inject + ThumbnailServiceWrapper thumbnailServiceWrapper; + @Inject + SettingsWrapper settingsWrapper; private Dataset dataset = new Dataset(); @@ -174,7 +180,7 @@ public enum DisplayMode { private Long ownerId; private Long versionId; private int selectedTabIndex; - private List newFiles = new ArrayList(); + private List newFiles = new ArrayList<>(); private DatasetVersion workingVersion; private int releaseRadio = 1; private int deaccessionRadio = 0; @@ -186,7 +192,7 @@ public enum DisplayMode { private String displayCitation; private String deaccessionForwardURLFor = ""; private String showVersionList = "false"; - private List