diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index b8c20da..c1e7712 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -26,7 +26,7 @@ jobs: environment-name: TEST init-shell: bash create-args: >- - python=3 pip iris xarray cdms2 genutil cdutil sphinx sphinx-issues --channel conda-forge + python=3 pip iris xarray sphinx sphinx-issues --channel conda-forge - name: Install eofs shell: bash -l {0} diff --git a/.github/workflows/tests-standard.yml b/.github/workflows/tests-standard.yml index c717891..2f41d1f 100644 --- a/.github/workflows/tests-standard.yml +++ b/.github/workflows/tests-standard.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11", "3.12"] os: [windows-latest, ubuntu-latest, macos-latest] fail-fast: false diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f1f276b..0d41e2b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] os: [ubuntu-latest, macos-latest] fail-fast: false @@ -23,7 +23,7 @@ jobs: environment-name: TEST init-shell: bash create-args: >- - python=${{ matrix.python-version }} pip pytest iris xarray cdms2 genutil cdutil --channel conda-forge + python=${{ matrix.python-version }} pip pytest iris xarray --channel conda-forge - name: Install eofs shell: bash -l {0} diff --git a/README.md b/README.md index 95f8136..ff9b4c9 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ eofs - EOF analysis in Python ============================= -[![Build Status](https://travis-ci.org/ajdawson/eofs.svg?branch=master)](https://travis-ci.org/ajdawson/eofs) [![DOI (paper)](https://img.shields.io/badge/DOI%20%28paper%29-10.5334%2Fjors.122-blue.svg)](http://doi.org/10.5334/jors.122) [![DOI (latest release)](https://zenodo.org/badge/20448/ajdawson/eofs.svg)](https://zenodo.org/badge/latestdoi/20448/ajdawson/eofs) +[![DOI (paper)](https://img.shields.io/badge/DOI%20%28paper%29-10.5334%2Fjors.122-blue.svg)](http://doi.org/10.5334/jors.122) [![DOI (latest release)](https://zenodo.org/badge/20448/ajdawson/eofs.svg)](https://zenodo.org/badge/latestdoi/20448/ajdawson/eofs) Overview @@ -14,7 +14,7 @@ Some of the key features are listed below: * Suitable for large data sets: computationally efficient for the large data sets typical of modern climate model output. * Transparent handling of missing values: missing values are removed automatically when computing EOFs and re-inserted into output fields. -* Meta-data preserving interfaces (optional): works with the iris data analysis package, xarray, or the cdms2 module (from UV-CDAT) to carry meta-data over from input fields to output. +* Meta-data preserving interfaces (optional): works with the iris data analysis package or xarray to carry meta-data over from input fields to output. * No Fortran dependencies: written in Python using the power of NumPy, no compilers required. @@ -22,7 +22,7 @@ Requirements ------------ eofs only requires the NumPy package (and setuptools to install). -In order to use the meta-data preserving interfaces one (or more) of cdms2 (part of [UV-CDAT](http://uvcdat.llnl.gov/)), [iris](http://scitools.org.uk/iris), or [xarray](http://xarray.pydata.org) is needed. +In order to use the meta-data preserving interfaces one (or both) of [iris](http://scitools.org.uk/iris) or [xarray](http://xarray.pydata.org) is needed. Documentation @@ -42,7 +42,7 @@ You can additionally cite the [Zenodo DOI](http://dx.doi.org/10.5281/zenodo.4687 Installation ------------ -eofs works on Python 2 or 3 on Linux, Windows or OSX. +eofs works on Python 3 on Linux, Windows or MacOS. The easiest way to install eofs is by using [conda](http://conda.pydata.org/docs/) or pip: conda install -c conda-forge eofs @@ -59,6 +59,6 @@ You can also install from the source distribution: Frequently asked questions -------------------------- -* **Do I need UV-CDAT/cdms2, iris or xarray to use eofs?** +* **Do I need iris or xarray to use eofs?** No. All the computation code uses NumPy only. - The cdms2 module, iris and xarray are only required for the meta-data preserving interfaces. + The iris and xarray modules are only required for the meta-data preserving interfaces. diff --git a/doc/api/eofs.cdms.rst b/doc/api/eofs.cdms.rst deleted file mode 100644 index cb258ab..0000000 --- a/doc/api/eofs.cdms.rst +++ /dev/null @@ -1,9 +0,0 @@ -:mod:`eofs.cdms` -================ - -.. default-role:: py:obj - -.. currentmodule:: eofs.cdms - -.. automodule:: eofs.cdms - :members: Eof diff --git a/doc/api/eofs.multivariate.cdms.rst b/doc/api/eofs.multivariate.cdms.rst deleted file mode 100644 index d5a91b7..0000000 --- a/doc/api/eofs.multivariate.cdms.rst +++ /dev/null @@ -1,9 +0,0 @@ -:mod:`eofs.multivariate.cdms` -============================= - -.. default-role:: py:obj - -.. currentmodule:: eofs.multivariate.cdms - -.. automodule:: eofs.multivariate.cdms - :members: MultivariateEof diff --git a/doc/api/eofs.tools.rst b/doc/api/eofs.tools.rst index 57e8f30..fa49764 100644 --- a/doc/api/eofs.tools.rst +++ b/doc/api/eofs.tools.rst @@ -16,16 +16,15 @@ :members: -`eofs.tools.cdms` +`eofs.tools.iris` ----------------- -.. automodule:: eofs.tools.cdms +.. automodule:: eofs.tools.iris :members: -`eofs.tools.iris` ------------------ +`eofs.tools.xarray` +------------------- -.. automodule:: eofs.tools.iris +.. automodule:: eofs.tools.xarray :members: - diff --git a/doc/api/index.rst b/doc/api/index.rst index fe50cb7..a0ca062 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -11,11 +11,9 @@ Available EOF solvers :nosignatures: eofs.standard.Eof - eofs.cdms.Eof eofs.iris.Eof eofs.xarray.Eof eofs.multivariate.standard.MultivariateEof - eofs.multivariate.cdms.MultivariateEof eofs.multivariate.iris.MultivariateEof @@ -26,10 +24,8 @@ Full documentation :maxdepth: 1 eofs.standard - eofs.cdms eofs.iris eofs.xarray eofs.tools eofs.multivariate.standard - eofs.multivariate.cdms eofs.multivariate.iris diff --git a/doc/devguide/gitwash/branch_dropdown.png b/doc/devguide/gitwash/branch_dropdown.png deleted file mode 100644 index 1bb7a57..0000000 Binary files a/doc/devguide/gitwash/branch_dropdown.png and /dev/null differ diff --git a/doc/devguide/gitwash/configure_git.rst b/doc/devguide/gitwash/configure_git.rst deleted file mode 100644 index f8a005f..0000000 --- a/doc/devguide/gitwash/configure_git.rst +++ /dev/null @@ -1,158 +0,0 @@ -.. _configure-git: - -=============== - Configure git -=============== - -.. _git-config-basic: - -Overview -======== - -Your personal git configurations are saved in the ``.gitconfig`` file in -your home directory. - -Here is an example ``.gitconfig`` file:: - - [user] - name = Your Name - email = you@yourdomain.example.com - - [alias] - ci = commit -a - co = checkout - st = status - stat = status - br = branch - wdiff = diff --color-words - - [core] - editor = vim - - [merge] - summary = true - -You can edit this file directly or you can use the ``git config --global`` -command:: - - git config --global user.name "Your Name" - git config --global user.email you@yourdomain.example.com - git config --global alias.ci "commit -a" - git config --global alias.co checkout - git config --global alias.st "status -a" - git config --global alias.stat "status -a" - git config --global alias.br branch - git config --global alias.wdiff "diff --color-words" - git config --global core.editor vim - git config --global merge.summary true - -To set up on another computer, you can copy your ``~/.gitconfig`` file, -or run the commands above. - -In detail -========= - -user.name and user.email ------------------------- - -It is good practice to tell git_ who you are, for labeling any changes -you make to the code. The simplest way to do this is from the command -line:: - - git config --global user.name "Your Name" - git config --global user.email you@yourdomain.example.com - -This will write the settings into your git configuration file, which -should now contain a user section with your name and email:: - - [user] - name = Your Name - email = you@yourdomain.example.com - -Of course you'll need to replace ``Your Name`` and ``you@yourdomain.example.com`` -with your actual name and email address. - -Aliases -------- - -You might well benefit from some aliases to common commands. - -For example, you might well want to be able to shorten ``git checkout`` -to ``git co``. Or you may want to alias ``git diff --color-words`` -(which gives a nicely formatted output of the diff) to ``git wdiff`` - -The following ``git config --global`` commands:: - - git config --global alias.ci "commit -a" - git config --global alias.co checkout - git config --global alias.st "status -a" - git config --global alias.stat "status -a" - git config --global alias.br branch - git config --global alias.wdiff "diff --color-words" - -will create an ``alias`` section in your ``.gitconfig`` file with contents -like this:: - - [alias] - ci = commit -a - co = checkout - st = status -a - stat = status -a - br = branch - wdiff = diff --color-words - -Editor ------- - -You may also want to make sure that your editor of choice is used :: - - git config --global core.editor vim - -Merging -------- - -To enforce summaries when doing merges (``~/.gitconfig`` file again):: - - [merge] - log = true - -Or from the command line:: - - git config --global merge.log true - -.. _fancy-log: - -Fancy log output ----------------- - -This is a very nice alias to get a fancy log output; it should go in the -``alias`` section of your ``.gitconfig`` file:: - - lg = log --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)[%an]%Creset' --abbrev-commit --date=relative - -You use the alias with:: - - git lg - -and it gives graph / text output something like this (but with color!):: - - * 6d8e1ee - (HEAD, origin/my-fancy-feature, my-fancy-feature) NF - a fancy file (45 minutes ago) [Matthew Brett] - * d304a73 - (origin/placeholder, placeholder) Merge pull request #48 from hhuuggoo/master (2 weeks ago) [Jonathan Terhorst] - |\ - | * 4aff2a8 - fixed bug 35, and added a test in test_bugfixes (2 weeks ago) [Hugo] - |/ - * a7ff2e5 - Added notes on discussion/proposal made during Data Array Summit. (2 weeks ago) [Corran Webster] - * 68f6752 - Initial implimentation of AxisIndexer - uses 'index_by' which needs to be changed to a call on an Axes object - this is all very sketchy right now. (2 weeks ago) [Corr - * 376adbd - Merge pull request #46 from terhorst/master (2 weeks ago) [Jonathan Terhorst] - |\ - | * b605216 - updated joshu example to current api (3 weeks ago) [Jonathan Terhorst] - | * 2e991e8 - add testing for outer ufunc (3 weeks ago) [Jonathan Terhorst] - | * 7beda5a - prevent axis from throwing an exception if testing equality with non-axis object (3 weeks ago) [Jonathan Terhorst] - | * 65af65e - convert unit testing code to assertions (3 weeks ago) [Jonathan Terhorst] - | * 956fbab - Merge remote-tracking branch 'upstream/master' (3 weeks ago) [Jonathan Terhorst] - | |\ - | |/ - -Thanks to Yury V. Zaytsev for posting it. - -.. include:: links.inc diff --git a/doc/devguide/gitwash/development_workflow.rst b/doc/devguide/gitwash/development_workflow.rst deleted file mode 100644 index 57be954..0000000 --- a/doc/devguide/gitwash/development_workflow.rst +++ /dev/null @@ -1,414 +0,0 @@ -.. _development-workflow: - -#################### -Development workflow -#################### - -You already have your own forked copy of the `eofs`_ repository, by -following :ref:`forking`. You have :ref:`set-up-fork`. You have configured -git by following :ref:`configure-git`. Now you are ready for some real work. - -Workflow summary -================ - -In what follows we'll refer to the upstream eofs ``master`` branch, as -"trunk". - -* Don't use your ``master`` branch for anything. Consider deleting it. -* When you are starting a new set of changes, fetch any changes from trunk, - and start a new *feature branch* from that. -* Make a new branch for each separable set of changes |emdash| "one task, one - branch" (`ipython git workflow`_). -* Name your branch for the purpose of the changes - e.g. - ``bugfix-for-issue-14`` or ``refactor-database-code``. -* If you can possibly avoid it, avoid merging trunk or any other branches into - your feature branch while you are working. -* If you do find yourself merging from trunk, consider :ref:`rebase-on-trunk` -* Ask for code review! - -This way of working helps to keep work well organized, with readable history. -This in turn makes it easier for project maintainers (that might be you) to see -what you've done, and why you did it. - -See `linux git workflow`_ and `ipython git workflow`_ for some explanation. - -Consider deleting your master branch -==================================== - -It may sound strange, but deleting your own ``master`` branch can help reduce -confusion about which branch you are on. See `deleting master on github`_ for -details. - -.. _update-mirror-trunk: - -Update the mirror of trunk -========================== - -First make sure you have done :ref:`linking-to-upstream`. - -From time to time you should fetch the upstream (trunk) changes from github:: - - git fetch upstream - -This will pull down any commits you don't have, and set the remote branches to -point to the right commit. For example, 'trunk' is the branch referred to by -(remote/branchname) ``upstream/master`` - and if there have been commits since -you last checked, ``upstream/master`` will change after you do the fetch. - -.. _make-feature-branch: - -Make a new feature branch -========================= - -When you are ready to make some changes to the code, you should start a new -branch. Branches that are for a collection of related edits are often called -'feature branches'. - -Making an new branch for each set of related changes will make it easier for -someone reviewing your branch to see what you are doing. - -Choose an informative name for the branch to remind yourself and the rest of us -what the changes in the branch are for. For example ``add-ability-to-fly``, or -``buxfix-for-issue-42``. - -:: - - # Update the mirror of trunk - git fetch upstream - # Make new feature branch starting at current trunk - git branch my-new-feature upstream/master - git checkout my-new-feature - -Generally, you will want to keep your feature branches on your public github_ -fork of `eofs`_. To do this, you `git push`_ this new branch up to your -github repo. Generally (if you followed the instructions in these pages, and by -default), git will have a link to your github repo, called ``origin``. You push -up to your own repo on github with:: - - git push origin my-new-feature - -In git >= 1.7 you can ensure that the link is correctly set by using the -``--set-upstream`` option:: - - git push --set-upstream origin my-new-feature - -From now on git will know that ``my-new-feature`` is related to the -``my-new-feature`` branch in the github repo. - -.. _edit-flow: - -The editing workflow -==================== - -Overview --------- - -:: - - # hack hack - git add my_new_file - git commit -am 'NF - some message' - git push - -In more detail --------------- - -#. Make some changes -#. See which files have changed with ``git status`` (see `git status`_). - You'll see a listing like this one:: - - # On branch ny-new-feature - # Changed but not updated: - # (use "git add ..." to update what will be committed) - # (use "git checkout -- ..." to discard changes in working directory) - # - # modified: README - # - # Untracked files: - # (use "git add ..." to include in what will be committed) - # - # INSTALL - no changes added to commit (use "git add" and/or "git commit -a") - -#. Check what the actual changes are with ``git diff`` (`git diff`_). -#. Add any new files to version control ``git add new_file_name`` (see - `git add`_). -#. To commit all modified files into the local copy of your repo,, do - ``git commit -am 'A commit message'``. Note the ``-am`` options to - ``commit``. The ``m`` flag just signals that you're going to type a - message on the command line. The ``a`` flag |emdash| you can just take on - faith |emdash| or see `why the -a flag?`_ |emdash| and the helpful use-case - description in the `tangled working copy problem`_. The `git commit`_ manual - page might also be useful. -#. To push the changes up to your forked repo on github, do a ``git - push`` (see `git push`_). - -Ask for your changes to be reviewed or merged -============================================= - -When you are ready to ask for someone to review your code and consider a merge: - -#. Go to the URL of your forked repo, say - ``http://github.com/your-user-name/eofs``. -#. Use the 'Switch Branches' dropdown menu near the top left of the page to - select the branch with your changes: - - .. image:: branch_dropdown.png - -#. Click on the 'Pull request' button: - - .. image:: pull_button.png - - Enter a title for the set of changes, and some explanation of what you've - done. Say if there is anything you'd like particular attention for - like a - complicated change or some code you are not happy with. - - If you don't think your request is ready to be merged, just say so in your - pull request message. This is still a good way of getting some preliminary - code review. - -Some other things you might want to do -====================================== - -Delete a branch on github -------------------------- - -:: - - git checkout master - # delete branch locally - git branch -D my-unwanted-branch - # delete branch on github - git push origin :my-unwanted-branch - -(Note the colon ``:`` before ``test-branch``. See also: -http://github.com/guides/remove-a-remote-branch - -Several people sharing a single repository ------------------------------------------- - -If you want to work on some stuff with other people, where you are all -committing into the same repository, or even the same branch, then just -share it via github. - -First fork eofs into your account, as from :ref:`forking`. - -Then, go to your forked repository github page, say -``http://github.com/your-user-name/eofs`` - -Click on the 'Admin' button, and add anyone else to the repo as a -collaborator: - - .. image:: pull_button.png - -Now all those people can do:: - - git clone git@githhub.com:your-user-name/eofs.git - -Remember that links starting with ``git@`` use the ssh protocol and are -read-write; links starting with ``git://`` are read-only. - -Your collaborators can then commit directly into that repo with the -usual:: - - git commit -am 'ENH - much better code' - git push origin master # pushes directly into your repo - -Explore your repository ------------------------ - -To see a graphical representation of the repository branches and -commits:: - - gitk --all - -To see a linear list of commits for this branch:: - - git log - -You can also look at the `network graph visualizer`_ for your github -repo. - -Finally the :ref:`fancy-log` ``lg`` alias will give you a reasonable text-based -graph of the repository. - -.. _rebase-on-trunk: - -Rebasing on trunk ------------------ - -Let's say you thought of some work you'd like to do. You -:ref:`update-mirror-trunk` and :ref:`make-feature-branch` called -``cool-feature``. At this stage trunk is at some commit, let's call it E. Now -you make some new commits on your ``cool-feature`` branch, let's call them A, B, -C. Maybe your changes take a while, or you come back to them after a while. In -the meantime, trunk has progressed from commit E to commit (say) G:: - - A---B---C cool-feature - / - D---E---F---G trunk - -At this stage you consider merging trunk into your feature branch, and you -remember that this here page sternly advises you not to do that, because the -history will get messy. Most of the time you can just ask for a review, and not -worry that trunk has got a little ahead. But sometimes, the changes in trunk -might affect your changes, and you need to harmonize them. In this situation -you may prefer to do a rebase. - -rebase takes your changes (A, B, C) and replays them as if they had been made to -the current state of ``trunk``. In other words, in this case, it takes the -changes represented by A, B, C and replays them on top of G. After the rebase, -your history will look like this:: - - A'--B'--C' cool-feature - / - D---E---F---G trunk - -See `rebase without tears`_ for more detail. - -To do a rebase on trunk:: - - # Update the mirror of trunk - git fetch upstream - # go to the feature branch - git checkout cool-feature - # make a backup in case you mess up - git branch tmp cool-feature - # rebase cool-feature onto trunk - git rebase --onto upstream/master upstream/master cool-feature - -In this situation, where you are already on branch ``cool-feature``, the last -command can be written more succinctly as:: - - git rebase upstream/master - -When all looks good you can delete your backup branch:: - - git branch -D tmp - -If it doesn't look good you may need to have a look at -:ref:`recovering-from-mess-up`. - -If you have made changes to files that have also changed in trunk, this may -generate merge conflicts that you need to resolve - see the `git rebase`_ man -page for some instructions at the end of the "Description" section. There is -some related help on merging in the git user manual - see `resolving a merge`_. - -.. _recovering-from-mess-up: - -Recovering from mess-ups ------------------------- - -Sometimes, you mess up merges or rebases. Luckily, in git it is -relatively straightforward to recover from such mistakes. - -If you mess up during a rebase:: - - git rebase --abort - -If you notice you messed up after the rebase:: - - # reset branch back to the saved point - git reset --hard tmp - -If you forgot to make a backup branch:: - - # look at the reflog of the branch - git reflog show cool-feature - - 8630830 cool-feature@{0}: commit: BUG: io: close file handles immediately - 278dd2a cool-feature@{1}: rebase finished: refs/heads/my-feature-branch onto 11ee694744f2552d - 26aa21a cool-feature@{2}: commit: BUG: lib: make seek_gzip_factory not leak gzip obj - ... - - # reset the branch to where it was before the botched rebase - git reset --hard cool-feature@{2} - -.. _rewriting-commit-history: - -Rewriting commit history ------------------------- - -.. note:: - - Do this only for your own feature branches. - -There's an embarrassing typo in a commit you made? Or perhaps the you -made several false starts you would like the posterity not to see. - -This can be done via *interactive rebasing*. - -Suppose that the commit history looks like this:: - - git log --oneline - eadc391 Fix some remaining bugs - a815645 Modify it so that it works - 2dec1ac Fix a few bugs + disable - 13d7934 First implementation - 6ad92e5 * masked is now an instance of a new object, MaskedConstant - 29001ed Add pre-nep for a copule of structured_array_extensions. - ... - -and ``6ad92e5`` is the last commit in the ``cool-feature`` branch. Suppose we -want to make the following changes: - -* Rewrite the commit message for ``13d7934`` to something more sensible. -* Combine the commits ``2dec1ac``, ``a815645``, ``eadc391`` into a single one. - -We do as follows:: - - # make a backup of the current state - git branch tmp HEAD - # interactive rebase - git rebase -i 6ad92e5 - -This will open an editor with the following text in it:: - - pick 13d7934 First implementation - pick 2dec1ac Fix a few bugs + disable - pick a815645 Modify it so that it works - pick eadc391 Fix some remaining bugs - - # Rebase 6ad92e5..eadc391 onto 6ad92e5 - # - # Commands: - # p, pick = use commit - # r, reword = use commit, but edit the commit message - # e, edit = use commit, but stop for amending - # s, squash = use commit, but meld into previous commit - # f, fixup = like "squash", but discard this commit's log message - # - # If you remove a line here THAT COMMIT WILL BE LOST. - # However, if you remove everything, the rebase will be aborted. - # - -To achieve what we want, we will make the following changes to it:: - - r 13d7934 First implementation - pick 2dec1ac Fix a few bugs + disable - f a815645 Modify it so that it works - f eadc391 Fix some remaining bugs - -This means that (i) we want to edit the commit message for -``13d7934``, and (ii) collapse the last three commits into one. Now we -save and quit the editor. - -Git will then immediately bring up an editor for editing the commit -message. After revising it, we get the output:: - - [detached HEAD 721fc64] FOO: First implementation - 2 files changed, 199 insertions(+), 66 deletions(-) - [detached HEAD 0f22701] Fix a few bugs + disable - 1 files changed, 79 insertions(+), 61 deletions(-) - Successfully rebased and updated refs/heads/my-feature-branch. - -and the history looks now like this:: - - 0f22701 Fix a few bugs + disable - 721fc64 ENH: Sophisticated feature - 6ad92e5 * masked is now an instance of a new object, MaskedConstant - -If it went wrong, recovery is again possible as explained :ref:`above -`. - -.. include:: links.inc diff --git a/doc/devguide/gitwash/following_latest.rst b/doc/devguide/gitwash/following_latest.rst deleted file mode 100644 index 0ef66c0..0000000 --- a/doc/devguide/gitwash/following_latest.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. _following-latest: - -============================= - Following the latest source -============================= - -These are the instructions if you just want to follow the latest -*eofs* source, but you don't need to do any development for now. - -The steps are: - -* :ref:`install-git` -* get local copy of the `eofs github`_ git repository -* update local copy from time to time - -Get the local copy of the code -============================== - -From the command line:: - - git clone git://github.com/ajdawson/eofs.git - -You now have a copy of the code tree in the new ``eofs`` directory. - -Updating the code -================= - -From time to time you may want to pull down the latest code. Do this with:: - - cd eofs - git pull - -The tree in ``eofs`` will now have the latest changes from the initial -repository. - -.. include:: links.inc diff --git a/doc/devguide/gitwash/forking_button.png b/doc/devguide/gitwash/forking_button.png deleted file mode 100644 index d0e0413..0000000 Binary files a/doc/devguide/gitwash/forking_button.png and /dev/null differ diff --git a/doc/devguide/gitwash/forking_hell.rst b/doc/devguide/gitwash/forking_hell.rst deleted file mode 100644 index 4d151cc..0000000 --- a/doc/devguide/gitwash/forking_hell.rst +++ /dev/null @@ -1,32 +0,0 @@ -.. _forking: - -====================================================== -Making your own copy (fork) of eofs -====================================================== - -You need to do this only once. The instructions here are very similar -to the instructions at http://help.github.com/forking/ |emdash| please see -that page for more detail. We're repeating some of it here just to give the -specifics for the `eofs`_ project, and to suggest some default names. - -Set up and configure a github account -===================================== - -If you don't have a github account, go to the github page, and make one. - -You then need to configure your account to allow write access |emdash| see -the ``Generating SSH keys`` help on `github help`_. - -Create your own forked copy of `eofs`_ -====================================================== - -#. Log into your github account. -#. Go to the `eofs`_ github home at `eofs github`_. -#. Click on the *fork* button: - - .. image:: forking_button.png - - Now, after a short pause and some 'Hardcore forking action', you - should find yourself at the home page for your own forked copy of `eofs`_. - -.. include:: links.inc diff --git a/doc/devguide/gitwash/git_development.rst b/doc/devguide/gitwash/git_development.rst deleted file mode 100644 index c5b910d..0000000 --- a/doc/devguide/gitwash/git_development.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. _git-development: - -===================== - Git for development -===================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - forking_hell - set_up_fork - configure_git - development_workflow - maintainer_workflow diff --git a/doc/devguide/gitwash/git_install.rst b/doc/devguide/gitwash/git_install.rst deleted file mode 100644 index 3be5149..0000000 --- a/doc/devguide/gitwash/git_install.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _install-git: - -============= - Install git -============= - -Overview -======== - -================ ============= -Debian / Ubuntu ``sudo apt-get install git`` -Fedora ``sudo yum install git`` -Windows Download and install msysGit_ -OS X Use the git-osx-installer_ -================ ============= - -In detail -========= - -See the git page for the most recent information. - -Have a look at the github install help pages available from `github help`_ - -There are good instructions here: http://book.git-scm.com/2_installing_git.html - -.. include:: links.inc diff --git a/doc/devguide/gitwash/git_intro.rst b/doc/devguide/gitwash/git_intro.rst deleted file mode 100644 index ff0de4f..0000000 --- a/doc/devguide/gitwash/git_intro.rst +++ /dev/null @@ -1,18 +0,0 @@ -============== - Introduction -============== - -These pages describe a git_ and github_ workflow for the `eofs`_ -project. - -There are several different workflows here, for different ways of -working with *eofs*. - -This is not a comprehensive git reference, it's just a workflow for our -own project. It's tailored to the github hosting service. You may well -find better or quicker ways of getting stuff done with git, but these -should get you started. - -For general resources for learning git, see :ref:`git-resources`. - -.. include:: links.inc diff --git a/doc/devguide/gitwash/git_links.inc b/doc/devguide/gitwash/git_links.inc deleted file mode 100644 index 82a72dd..0000000 --- a/doc/devguide/gitwash/git_links.inc +++ /dev/null @@ -1,61 +0,0 @@ -.. This (-*- rst -*-) format file contains commonly used link targets - and name substitutions. It may be included in many files, - therefore it should only contain link targets and name - substitutions. Try grepping for "^\.\. _" to find plausible - candidates for this list. - -.. NOTE: reST targets are - __not_case_sensitive__, so only one target definition is needed for - nipy, NIPY, Nipy, etc... - -.. git stuff -.. _git: http://git-scm.com/ -.. _github: http://github.com -.. _github help: http://help.github.com -.. _msysgit: http://code.google.com/p/msysgit/downloads/list -.. _git-osx-installer: http://code.google.com/p/git-osx-installer/downloads/list -.. _subversion: http://subversion.tigris.org/ -.. _git cheat sheet: http://github.com/guides/git-cheat-sheet -.. _pro git book: http://progit.org/ -.. _git svn crash course: http://git-scm.com/course/svn.html -.. _learn.github: http://learn.github.com/ -.. _network graph visualizer: http://github.com/blog/39-say-hello-to-the-network-graph-visualizer -.. _git user manual: http://schacon.github.com/git/user-manual.html -.. _git tutorial: http://schacon.github.com/git/gittutorial.html -.. _git community book: http://book.git-scm.com/ -.. _git ready: http://www.gitready.com/ -.. _git casts: http://www.gitcasts.com/ -.. _Fernando's git page: http://www.fperez.org/py4science/git.html -.. _git magic: http://www-cs-students.stanford.edu/~blynn/gitmagic/index.html -.. _git concepts: http://www.eecs.harvard.edu/~cduan/technical/git/ -.. _git clone: http://schacon.github.com/git/git-clone.html -.. _git checkout: http://schacon.github.com/git/git-checkout.html -.. _git commit: http://schacon.github.com/git/git-commit.html -.. _git push: http://schacon.github.com/git/git-push.html -.. _git pull: http://schacon.github.com/git/git-pull.html -.. _git add: http://schacon.github.com/git/git-add.html -.. _git status: http://schacon.github.com/git/git-status.html -.. _git diff: http://schacon.github.com/git/git-diff.html -.. _git log: http://schacon.github.com/git/git-log.html -.. _git branch: http://schacon.github.com/git/git-branch.html -.. _git remote: http://schacon.github.com/git/git-remote.html -.. _git rebase: http://schacon.github.com/git/git-rebase.html -.. _git config: http://schacon.github.com/git/git-config.html -.. _why the -a flag?: http://www.gitready.com/beginner/2009/01/18/the-staging-area.html -.. _git staging area: http://www.gitready.com/beginner/2009/01/18/the-staging-area.html -.. _tangled working copy problem: http://tomayko.com/writings/the-thing-about-git -.. _git management: http://kerneltrap.org/Linux/Git_Management -.. _linux git workflow: http://www.mail-archive.com/dri-devel@lists.sourceforge.net/msg39091.html -.. _git parable: http://tom.preston-werner.com/2009/05/19/the-git-parable.html -.. _git foundation: http://matthew-brett.github.com/pydagogue/foundation.html -.. _deleting master on github: http://matthew-brett.github.com/pydagogue/gh_delete_master.html -.. _rebase without tears: http://matthew-brett.github.com/pydagogue/rebase_without_tears.html -.. _resolving a merge: http://schacon.github.com/git/user-manual.html#resolving-a-merge -.. _ipython git workflow: http://mail.scipy.org/pipermail/ipython-dev/2010-October/006746.html - -.. other stuff -.. _python: http://www.python.org - -.. |emdash| unicode:: U+02014 - -.. vim: ft=rst diff --git a/doc/devguide/gitwash/git_resources.rst b/doc/devguide/gitwash/git_resources.rst deleted file mode 100644 index d18b0ef..0000000 --- a/doc/devguide/gitwash/git_resources.rst +++ /dev/null @@ -1,59 +0,0 @@ -.. _git-resources: - -============= -git resources -============= - -Tutorials and summaries -======================= - -* `github help`_ has an excellent series of how-to guides. -* `learn.github`_ has an excellent series of tutorials -* The `pro git book`_ is a good in-depth book on git. -* A `git cheat sheet`_ is a page giving summaries of common commands. -* The `git user manual`_ -* The `git tutorial`_ -* The `git community book`_ -* `git ready`_ |emdash| a nice series of tutorials -* `git casts`_ |emdash| video snippets giving git how-tos. -* `git magic`_ |emdash| extended introduction with intermediate detail -* The `git parable`_ is an easy read explaining the concepts behind git. -* `git foundation`_ expands on the `git parable`_. -* Fernando Perez' git page |emdash| `Fernando's git page`_ |emdash| many - links and tips -* A good but technical page on `git concepts`_ -* `git svn crash course`_: git for those of us used to subversion_ - -Advanced git workflow -===================== - -There are many ways of working with git; here are some posts on the -rules of thumb that other projects have come up with: - -* Linus Torvalds on `git management`_ -* Linus Torvalds on `linux git workflow`_ . Summary; use the git tools - to make the history of your edits as clean as possible; merge from - upstream edits as little as possible in branches where you are doing - active development. - -Manual pages online -=================== - -You can get these on your own machine with (e.g) ``git help push`` or -(same thing) ``git push --help``, but, for convenience, here are the -online manual pages for some common commands: - -* `git add`_ -* `git branch`_ -* `git checkout`_ -* `git clone`_ -* `git commit`_ -* `git config`_ -* `git diff`_ -* `git log`_ -* `git pull`_ -* `git push`_ -* `git remote`_ -* `git status`_ - -.. include:: links.inc diff --git a/doc/devguide/gitwash/index.rst b/doc/devguide/gitwash/index.rst deleted file mode 100644 index 187810a..0000000 --- a/doc/devguide/gitwash/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. _using-git: - -Working with *eofs* source code -================================================ - -Contents: - -.. toctree:: - :maxdepth: 2 - - git_intro - git_install - following_latest - git_development - git_resources diff --git a/doc/devguide/gitwash/known_projects.inc b/doc/devguide/gitwash/known_projects.inc deleted file mode 100644 index 1761d97..0000000 --- a/doc/devguide/gitwash/known_projects.inc +++ /dev/null @@ -1,41 +0,0 @@ -.. Known projects - -.. PROJECTNAME placeholders -.. _PROJECTNAME: http://nipy.org -.. _`PROJECTNAME github`: https://github.com/nipy -.. _`PROJECTNAME mailing list`: https://mail.python.org/mailman/listinfo/neuroimaging - -.. numpy -.. _numpy: http://www.numpy.org -.. _`numpy github`: https://github.com/numpy/numpy -.. _`numpy mailing list`: http://mail.scipy.org/mailman/listinfo/numpy-discussion - -.. scipy -.. _scipy: https://www.scipy.org -.. _`scipy github`: https://github.com/scipy/scipy -.. _`scipy mailing list`: http://mail.scipy.org/mailman/listinfo/scipy-dev - -.. nipy -.. _nipy: http://nipy.org/nipy -.. _`nipy github`: https://github.com/nipy/nipy -.. _`nipy mailing list`: https://mail.python.org/mailman/listinfo/neuroimaging - -.. ipython -.. _ipython: https://ipython.org -.. _`ipython github`: https://github.com/ipython/ipython -.. _`ipython mailing list`: http://mail.scipy.org/mailman/listinfo/IPython-dev - -.. dipy -.. _dipy: http://nipy.org/dipy -.. _`dipy github`: https://github.com/Garyfallidis/dipy -.. _`dipy mailing list`: https://mail.python.org/mailman/listinfo/neuroimaging - -.. nibabel -.. _nibabel: http://nipy.org/nibabel -.. _`nibabel github`: https://github.com/nipy/nibabel -.. _`nibabel mailing list`: https://mail.python.org/mailman/listinfo/neuroimaging - -.. marsbar -.. _marsbar: http://marsbar.sourceforge.net -.. _`marsbar github`: https://github.com/matthew-brett/marsbar -.. _`MarsBaR mailing list`: https://lists.sourceforge.net/lists/listinfo/marsbar-users diff --git a/doc/devguide/gitwash/links.inc b/doc/devguide/gitwash/links.inc deleted file mode 100644 index 20f4dcf..0000000 --- a/doc/devguide/gitwash/links.inc +++ /dev/null @@ -1,4 +0,0 @@ -.. compiling links file -.. include:: known_projects.inc -.. include:: this_project.inc -.. include:: git_links.inc diff --git a/doc/devguide/gitwash/maintainer_workflow.rst b/doc/devguide/gitwash/maintainer_workflow.rst deleted file mode 100644 index b95ac52..0000000 --- a/doc/devguide/gitwash/maintainer_workflow.rst +++ /dev/null @@ -1,96 +0,0 @@ -.. _maintainer-workflow: - -################### -Maintainer workflow -################### - -This page is for maintainers |emdash| those of us who merge our own or other -peoples' changes into the upstream repository. - -Being as how you're a maintainer, you are completely on top of the basic stuff -in :ref:`development-workflow`. - -The instructions in :ref:`linking-to-upstream` add a remote that has read-only -access to the upstream repo. Being a maintainer, you've got read-write access. - -It's good to have your upstream remote have a scary name, to remind you that -it's a read-write remote:: - - git remote add upstream-rw git@github.com:ajdawson/eofs.git - git fetch upstream-rw - -******************* -Integrating changes -******************* - -Let's say you have some changes that need to go into trunk -(``upstream-rw/master``). - -The changes are in some branch that you are currently on. For example, you are -looking at someone's changes like this:: - - git remote add someone git://github.com/someone/eofs.git - git fetch someone - git branch cool-feature --track someone/cool-feature - git checkout cool-feature - -So now you are on the branch with the changes to be incorporated upstream. The -rest of this section assumes you are on this branch. - -A few commits -============= - -If there are only a few commits, consider rebasing to upstream:: - - # Fetch upstream changes - git fetch upstream-rw - # rebase - git rebase upstream-rw/master - -Remember that, if you do a rebase, and push that, you'll have to close any -github pull requests manually, because github will not be able to detect the -changes have already been merged. - -A long series of commits -======================== - -If there are a longer series of related commits, consider a merge instead:: - - git fetch upstream-rw - git merge --no-ff upstream-rw/master - -The merge will be detected by github, and should close any related pull requests -automatically. - -Note the ``--no-ff`` above. This forces git to make a merge commit, rather than -doing a fast-forward, so that these set of commits branch off trunk then rejoin -the main history with a merge, rather than appearing to have been made directly -on top of trunk. - -Check the history -================= - -Now, in either case, you should check that the history is sensible and you have -the right commits:: - - git log --oneline --graph - git log -p upstream-rw/master.. - -The first line above just shows the history in a compact way, with a text -representation of the history graph. The second line shows the log of commits -excluding those that can be reached from trunk (``upstream-rw/master``), and -including those that can be reached from current HEAD (implied with the ``..`` -at the end). So, it shows the commits unique to this branch compared to trunk. -The ``-p`` option shows the diff for these commits in patch form. - -Push to trunk -============= - -:: - - git push upstream-rw my-new-feature:master - -This pushes the ``my-new-feature`` branch in this repository to the ``master`` -branch in the ``upstream-rw`` repository. - -.. include:: links.inc diff --git a/doc/devguide/gitwash/pull_button.png b/doc/devguide/gitwash/pull_button.png deleted file mode 100644 index e503168..0000000 Binary files a/doc/devguide/gitwash/pull_button.png and /dev/null differ diff --git a/doc/devguide/gitwash/set_up_fork.rst b/doc/devguide/gitwash/set_up_fork.rst deleted file mode 100644 index 6720d1e..0000000 --- a/doc/devguide/gitwash/set_up_fork.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. _set-up-fork: - -================== - Set up your fork -================== - -First you follow the instructions for :ref:`forking`. - -Overview -======== - -:: - - git clone git@github.com:your-user-name/eofs.git - cd eofs - git remote add upstream git://github.com/ajdawson/eofs.git - -In detail -========= - -Clone your fork ---------------- - -#. Clone your fork to the local computer with ``git clone - git@github.com:your-user-name/eofs.git`` -#. Investigate. Change directory to your new repo: ``cd eofs``. Then - ``git branch -a`` to show you all branches. You'll get something - like:: - - * master - remotes/origin/master - - This tells you that you are currently on the ``master`` branch, and - that you also have a ``remote`` connection to ``origin/master``. - What remote repository is ``remote/origin``? Try ``git remote -v`` to - see the URLs for the remote. They will point to your github fork. - - Now you want to connect to the upstream `eofs github`_ repository, so - you can merge in changes from trunk. - -.. _linking-to-upstream: - -Linking your repository to the upstream repo --------------------------------------------- - -:: - - cd eofs - git remote add upstream git://github.com/ajdawson/eofs.git - -``upstream`` here is just the arbitrary name we're using to refer to the -main `eofs`_ repository at `eofs github`_. - -Note that we've used ``git://`` for the URL rather than ``git@``. The -``git://`` URL is read only. This means we that we can't accidentally -(or deliberately) write to the upstream repo, and we are only going to -use it to merge into our own code. - -Just for your own satisfaction, show yourself that you now have a new -'remote', with ``git remote -v show``, giving you something like:: - - upstream git://github.com/ajdawson/eofs.git (fetch) - upstream git://github.com/ajdawson/eofs.git (push) - origin git@github.com:your-user-name/eofs.git (fetch) - origin git@github.com:your-user-name/eofs.git (push) - -.. include:: links.inc diff --git a/doc/devguide/gitwash/this_project.inc b/doc/devguide/gitwash/this_project.inc deleted file mode 100644 index 0fe5fd7..0000000 --- a/doc/devguide/gitwash/this_project.inc +++ /dev/null @@ -1,3 +0,0 @@ -.. eofs -.. _`eofs`: http://ajdawson.github.io/eofs -.. _`eofs github`: http://github.com/ajdawson/eofs diff --git a/doc/devguide/gitwash_dumper.py b/doc/devguide/gitwash_dumper.py deleted file mode 100644 index ad431ff..0000000 --- a/doc/devguide/gitwash_dumper.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python -''' Checkout gitwash repo into directory and do search replace on name ''' - -from __future__ import (absolute_import, division, print_function) - -import os -from os.path import join as pjoin -import shutil -import sys -import re -import glob -import fnmatch -import tempfile -from subprocess import call -from optparse import OptionParser - -verbose = False - - -def clone_repo(url, branch): - cwd = os.getcwd() - tmpdir = tempfile.mkdtemp() - try: - cmd = 'git clone %s %s' % (url, tmpdir) - call(cmd, shell=True) - os.chdir(tmpdir) - cmd = 'git checkout %s' % branch - call(cmd, shell=True) - except: - shutil.rmtree(tmpdir) - raise - finally: - os.chdir(cwd) - return tmpdir - - -def cp_files(in_path, globs, out_path): - try: - os.makedirs(out_path) - except OSError: - pass - out_fnames = [] - for in_glob in globs: - in_glob_path = pjoin(in_path, in_glob) - for in_fname in glob.glob(in_glob_path): - out_fname = in_fname.replace(in_path, out_path) - pth, _ = os.path.split(out_fname) - if not os.path.isdir(pth): - os.makedirs(pth) - shutil.copyfile(in_fname, out_fname) - out_fnames.append(out_fname) - return out_fnames - - -def filename_search_replace(sr_pairs, filename, backup=False): - ''' Search and replace for expressions in files - - ''' - with open(filename, 'rt') as in_fh: - in_txt = in_fh.read(-1) - out_txt = in_txt[:] - for in_exp, out_exp in sr_pairs: - in_exp = re.compile(in_exp) - out_txt = in_exp.sub(out_exp, out_txt) - if in_txt == out_txt: - return False - with open(filename, 'wt') as out_fh: - out_fh.write(out_txt) - if backup: - with open(filename + '.bak', 'wt') as bak_fh: - bak_fh.write(in_txt) - return True - - -def copy_replace(replace_pairs, - repo_path, - out_path, - cp_globs=('*',), - rep_globs=('*',), - renames = ()): - out_fnames = cp_files(repo_path, cp_globs, out_path) - renames = [(re.compile(in_exp), out_exp) for in_exp, out_exp in renames] - fnames = [] - for rep_glob in rep_globs: - fnames += fnmatch.filter(out_fnames, rep_glob) - if verbose: - print('\n'.join(fnames)) - for fname in fnames: - filename_search_replace(replace_pairs, fname, False) - for in_exp, out_exp in renames: - new_fname, n = in_exp.subn(out_exp, fname) - if n: - os.rename(fname, new_fname) - break - - -def make_link_targets(proj_name, - user_name, - repo_name, - known_link_fname, - out_link_fname, - url=None, - ml_url=None): - """ Check and make link targets - - If url is None or ml_url is None, check if there are links present for these - in `known_link_fname`. If not, raise error. The check is: - - Look for a target `proj_name`. - Look for a target `proj_name` + ' mailing list' - - Also, look for a target `proj_name` + 'github'. If this exists, don't write - this target into the new file below. - - If we are writing any of the url, ml_url, or github address, then write new - file with these links, of form: - - .. _`proj_name` - .. _`proj_name`: url - .. _`proj_name` mailing list: url - """ - with open(known_link_fname, 'rt') as link_fh: - link_contents = link_fh.readlines() - have_url = not url is None - have_ml_url = not ml_url is None - have_gh_url = None - for line in link_contents: - if not have_url: - match = re.match(r'..\s+_`%s`:\s+' % proj_name, line) - if match: - have_url = True - if not have_ml_url: - match = re.match(r'..\s+_`%s mailing list`:\s+' % proj_name, line) - if match: - have_ml_url = True - if not have_gh_url: - match = re.match(r'..\s+_`%s github`:\s+' % proj_name, line) - if match: - have_gh_url = True - if not have_url or not have_ml_url: - raise RuntimeError('Need command line or known project ' - 'and / or mailing list URLs') - lines = [] - if not url is None: - lines.append('.. _`%s`: %s\n' % (proj_name, url)) - if not have_gh_url: - gh_url = 'http://github.com/%s/%s\n' % (user_name, repo_name) - lines.append('.. _`%s github`: %s\n' % (proj_name, gh_url)) - if not ml_url is None: - lines.append('.. _`%s mailing list`: %s\n' % (proj_name, ml_url)) - if len(lines) == 0: - # Nothing to do - return - # A neat little header line - lines = ['.. %s\n' % proj_name] + lines - with open(out_link_fname, 'wt') as out_links: - out_links.writelines(lines) - - -USAGE = ''' - -If not set with options, the repository name is the same as the - -If not set with options, the main github user is the same as the -repository name.''' - - -GITWASH_CENTRAL = 'git://github.com/matthew-brett/gitwash.git' -GITWASH_BRANCH = 'master' - - -def main(): - parser = OptionParser() - parser.set_usage(parser.get_usage().strip() + USAGE) - parser.add_option("--repo-name", dest="repo_name", - help="repository name - e.g. nitime", - metavar="REPO_NAME") - parser.add_option("--github-user", dest="main_gh_user", - help="github username for main repo - e.g fperez", - metavar="MAIN_GH_USER") - parser.add_option("--gitwash-url", dest="gitwash_url", - help="URL to gitwash repository - default %s" - % GITWASH_CENTRAL, - default=GITWASH_CENTRAL, - metavar="GITWASH_URL") - parser.add_option("--gitwash-branch", dest="gitwash_branch", - help="branch in gitwash repository - default %s" - % GITWASH_BRANCH, - default=GITWASH_BRANCH, - metavar="GITWASH_BRANCH") - parser.add_option("--source-suffix", dest="source_suffix", - help="suffix of ReST source files - default '.rst'", - default='.rst', - metavar="SOURCE_SUFFIX") - parser.add_option("--project-url", dest="project_url", - help="URL for project web pages", - default=None, - metavar="PROJECT_URL") - parser.add_option("--project-ml-url", dest="project_ml_url", - help="URL for project mailing list", - default=None, - metavar="PROJECT_ML_URL") - (options, args) = parser.parse_args() - if len(args) < 2: - parser.print_help() - sys.exit() - out_path, project_name = args - if options.repo_name is None: - options.repo_name = project_name - if options.main_gh_user is None: - options.main_gh_user = options.repo_name - repo_path = clone_repo(options.gitwash_url, options.gitwash_branch) - try: - copy_replace((('PROJECTNAME', project_name), - ('REPONAME', options.repo_name), - ('MAIN_GH_USER', options.main_gh_user)), - repo_path, - out_path, - cp_globs=(pjoin('gitwash', '*'),), - rep_globs=('*.rst',), - renames=(('\.rst$', options.source_suffix),)) - make_link_targets(project_name, - options.main_gh_user, - options.repo_name, - pjoin(out_path, 'gitwash', 'known_projects.inc'), - pjoin(out_path, 'gitwash', 'this_project.inc'), - options.project_url, - options.project_ml_url) - finally: - shutil.rmtree(repo_path) - - -if __name__ == '__main__': - main() diff --git a/doc/devguide/gitwash_wrapper.sh b/doc/devguide/gitwash_wrapper.sh deleted file mode 100755 index 3875cf4..0000000 --- a/doc/devguide/gitwash_wrapper.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# -# Run gitwash_dumper.py and edit the output to remove the email patching -# page which is not relevant to this project. -# - -set -u -set -e - -readonly PROJECT_NAME="eofs" -readonly REPO_NAME="eofs" -readonly GITHUB_USER="ajdawson" -readonly PROJECT_URL="http://ajdawson.github.io/eofs" -readonly OUTPUT_DIRECTORY="./" -readonly GITWASH_DUMPER="./gitwash_dumper.py" - -# Use the gitwash script to refresh the documentation. -python "$GITWASH_DUMPER" "$OUTPUT_DIRECTORY" "$PROJECT_NAME" \ - --repo-name="$REPO_NAME" \ - --github-user="$GITHUB_USER" \ - --project-url="$PROJECT_URL" \ - --project-ml-url="NONE" - -# Remove the patching section of the gitwash guide. -rm -f "${OUTPUT_DIRECTORY}/gitwash/patching.rst" -sed -i '/patching/d' "${OUTPUT_DIRECTORY}/gitwash/index.rst" - -# Remove references to the project mailing list in the gitwash guide. -sed -i '/mailing list/d' "${OUTPUT_DIRECTORY}/gitwash/this_project.inc" -sed -i '/mailing list/d' "${OUTPUT_DIRECTORY}/gitwash/development_workflow.rst" - -# Remove all trailing whitespace and trailing blank lines from the downloaded -# gitwash guide restructured text sources: -sed -i 's/[[:space:]]*$//' "${OUTPUT_DIRECTORY}"/gitwash/*.{rst,inc} -sed -i -e :a -e '/^\n*$/{$d;N;ba' -e '}' "${OUTPUT_DIRECTORY}"/gitwash/*.{rst,inc} diff --git a/doc/devguide/index.rst b/doc/devguide/index.rst index 7579b60..82d7e0b 100644 --- a/doc/devguide/index.rst +++ b/doc/devguide/index.rst @@ -6,5 +6,4 @@ This guide is for those who want to contribute to the development of `eofs`. .. toctree:: :maxdepth: 2 - gitwash/index testing diff --git a/doc/devguide/testing.rst b/doc/devguide/testing.rst index 26d4b6c..6008622 100644 --- a/doc/devguide/testing.rst +++ b/doc/devguide/testing.rst @@ -5,16 +5,16 @@ The package comes with a comprehensive set of tests to make sure it is working c The tests can be run against an installed version of `eofs` or against the current source tree. Testing against the source tree is handy during development when quick iteration is required, but for most other cases testing against the installed version is more suitable. -Running the test suite requires pytest_ and pycodestyle_ to be installed. +Running the test suite requires pytest_ to be installed. The test suite will function as long as the minimum dependencies for the package are installed, but some tests will be skipped if they require optional dependencies that are not present. -To run the full test suite you need to have the optional dependencies `cdms2` (from UV-CDAT_), iris_, and xarray_ installed. +To run the full test suite you need to have the optional dependencies iris_ and xarray_ installed. Testing against the current source tree --------------------------------------- Testing the current source is straightforward, from the source directory run:: - pytest + python -m pytest This will perform verbose testing of the current source tree and print a summary at the end. @@ -27,23 +27,19 @@ First you need to install `eofs` into your current Python environment:: .. code-block:: shell cd eofs/ - python setup.py install + python -m pip install ".[iris,xarray]" Then create a directory somewhere else without any Python code in it and run ``pytest`` from there:: .. code-block:: shell mkdir $HOME/eofs-test-dir && cd $HOME/eofs-test-dir - pytest --pyargs eofs + python -m pytest --pyargs eofs This will run the tests on the version of `eofs` you just installed. .. _pytest: https://docs.pytest.org/en/latest/ -.. _pycodestyle: https://pypi.python.org/pypi/pycodestyle - -.. _UV-CDAT: http://uv-cdat.llnl.gov - .. _iris: http://scitools.org.uk/iris .. _xarray: http://xarray.pydata.org diff --git a/doc/examples/cdms_examples_index.rst b/doc/examples/cdms_examples_index.rst deleted file mode 100644 index 2ad6183..0000000 --- a/doc/examples/cdms_examples_index.rst +++ /dev/null @@ -1,8 +0,0 @@ -cdms interface examples -======================== - -.. toctree:: - :maxdepth: 1 - - nao_cdms.rst - elnino_cdms.rst diff --git a/doc/examples/elnino_cdms.rst b/doc/examples/elnino_cdms.rst deleted file mode 100644 index d25a488..0000000 --- a/doc/examples/elnino_cdms.rst +++ /dev/null @@ -1,9 +0,0 @@ -El |Nino| -========= - - -.. plot:: example_code/cdms/sst_example.py - -.. literalinclude:: ../example_code/cdms/sst_example.py - -.. |Nino| unicode:: Ni U+00F1 o diff --git a/doc/examples/index.rst b/doc/examples/index.rst index 1353b83..8d34662 100644 --- a/doc/examples/index.rst +++ b/doc/examples/index.rst @@ -6,7 +6,6 @@ Examples :maxdepth: 2 standard_examples_index - cdms_examples_index iris_examples_index xarray_examples_index diff --git a/doc/examples/nao_cdms.rst b/doc/examples/nao_cdms.rst deleted file mode 100644 index 350cb0d..0000000 --- a/doc/examples/nao_cdms.rst +++ /dev/null @@ -1,7 +0,0 @@ -North Atlantic Oscillation -========================== - - -.. plot:: example_code/cdms/hgt_example.py - -.. literalinclude:: ../example_code/cdms/hgt_example.py diff --git a/doc/index.rst b/doc/index.rst index 9c274d4..ed3a08c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -28,7 +28,7 @@ Download and installation ------------------------- The core of the package runs on Python 2 or 3, on Linux, Windows or OSX; basically anywhere Python+NumPy are available. -The :ref:`cdms `, :ref:`iris `, and :ref:`xarray ` interfaces are available on all platforms where their respective supporting packages UV-CDAT_, iris_, and xarray_ can be installed. +The :ref:`iris `, and :ref:`xarray ` interfaces are available on all platforms where their respective supporting packages iris_ and xarray_ can be installed. `eofs` can be installed for all platforms using conda_:: @@ -50,7 +50,7 @@ You can also check out the source code for the development version from the `git Getting started --------------- -`eofs` provides various interfaces for EOF analysis: a *standard* interface for analysing data contained in either `numpy` arrays or `dask` arrays, suitable for any data set; and meta-data interfaces suitable for analysing data read from self-describing files, using the `cdms2`, `iris`, or `xarray` packages. +`eofs` provides various interfaces for EOF analysis: a *standard* interface for analysing data contained in either `numpy` arrays or `dask` arrays, suitable for any data set; and meta-data interfaces suitable for analysing data read from self-describing files, using the `iris` or `xarray` packages. All the interfaces support the same set of operations. Regardless of which interface you use, the basic usage is the same. @@ -81,8 +81,6 @@ Requirements This package requires as a minimum that you have numpy_ available, and requires setuptools_ for installation. The dask_ package is an optional dependency that will be used if `dask.array` is available and `dask` arrays are provided to the solver. -The `eofs.cdms` meta-data enabled interface can only be used if the `cdms2` module is available. -This module is distributed as part of the UV-CDAT_ project. The `eofs.iris` meta-data enabled interface can only be used if the iris_ package is available at version 1.2 or higher. The `eofs.xarray` meta-data enabled interface can only be used if the xarray_ package is installed. @@ -102,8 +100,6 @@ Bug reports and feature requests can be filed using the Github issues_ system. If you would like to contribute code or documentation please see the :doc:`devguide/index`. -.. _UV-CDAT: http://uv-cdat.llnl.gov - .. _dask: https://www.dask.org/ .. _iris: http://scitools.org.uk/iris diff --git a/doc/userguide/index.rst b/doc/userguide/index.rst index e225148..3a55b29 100644 --- a/doc/userguide/index.rst +++ b/doc/userguide/index.rst @@ -10,5 +10,3 @@ User Guide usage multivariate method - speed - diff --git a/doc/userguide/interfaces.rst b/doc/userguide/interfaces.rst index aa53aa2..5666198 100644 --- a/doc/userguide/interfaces.rst +++ b/doc/userguide/interfaces.rst @@ -10,8 +10,6 @@ Solver interfaces ========= ===================================================================== Interface Type of input/output data ========= ===================================================================== -cdms Data read from a self-describing data format stored in a `cdms2` - variable. iris Data read from a self-describing data format stored in an `iris` cube. xarray Data stored in an `xarray.DataArray`. @@ -20,14 +18,6 @@ standard Other data, stored in a `numpy.ndarray`, a `numpy.ma.MaskedArray`, ========= ===================================================================== -.. _cdms-interface: - -cdms interface --------------- - -The `eofs.cdms` interface works with cdms variables, which are the core data container used by UV-CDAT_. A cdms variable has meta-data associated with it, including dimensions, which are understood by the `eofs.cdms.Eof` solver interface. The outputs of the `eofs.cdms.Eof` solver are cdms variables with meta-data, which can be written straight to a netCDF file using cdms, or used with other parts of the UV-CDAT framework. - - .. _iris-interface: Iris interface @@ -54,8 +44,6 @@ The standard interface works with numpy_ arrays or dask_ arrays, which makes the .. _iris: https://scitools-iris.readthedocs.io/en/stable/ -.. _UV-CDAT: https://uv-cdat.llnl.gov/ - .. _xarray: https://docs.xarray.dev/en/stable/ .. _dask: https://www.dask.org/ diff --git a/doc/userguide/speed.rst b/doc/userguide/speed.rst deleted file mode 100644 index 02f05f5..0000000 --- a/doc/userguide/speed.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. default-role:: py:obj - -Speeding up the solvers -======================= - -All the solvers rely on the `numpy.linalg.svd` function to compute the EOF solution. If you obtained numpy_ from you package manager it is likely that this function uses standard LAPACK_/BLAS_ routines, which are single threaded. It is possible to get a large performance boost from the code by using a version of numpy that is built with an optimised linear algebra library. The following options all allow a performace boost for the solvers in `eofs`: - -NumPy built with the Intel Math Kernel Library (MKL) ----------------------------------------------------- - -Intel's MKL provides highly optimised BLAS and LAPACK routines which can take advantage of multicore processors. -If you have access to Intel's MKL you can build a numpy_ library using it following the instructions from intel_. -It is actually not that hard to do and can provide a great performance boost. - -Numpy built with the AMD Core Math Library (ACML) -------------------------------------------------- - -AMD's ACML provides optimised BLAS and LAPACK routines in both single and multi-threaded versions. The ACML itself is free to download. -You will have to build CBLAS first as ACML only includes Fortran interfaces. -The following instructions worked for me: - -1. download CBLAS from the link at https://www.netlib.org/blas/ - -2. edit Makefile.LINUX and change BLLIB to point to your libacml.so or libacml_mp.so - -3. copy or link this make file to Makefile.in and build CBLAS - -4. copy the resulting cblas library to libcblas.a in the same directory as the ACML library - -5. download a stable version fo the numpy source code from https://github.com/numpy/numpy - -6. create a site.cfg in the numpy source tree (copy site.cfg.example) and add:: - - [blas] - blas_libs = cblas, acml - library_dirs = /path-to-acml-and-cblas/lib - include_dirs = /path-to-acml-and-cblas/include - - [lapack] - language = f77 - lapack_libs = acml - library_dirs = /path-to-acml-and-cblas/lib - include_dirs = /path-to-acml-and-cblas/include - -7. build and install numpy as normal - -Pre-built options ------------------ - -Anaconda -~~~~~~~~ - -Continuum IO's Anaconda product has the option of using MKL optimised components. - -Canopy (Express) -~~~~~~~~~~~~~~~~ - -Enthought provides MKL linked packages in its Canopy product. I don't know if the free version Canopy Express also includes MKL linked libraries, but it may be worth investigating. - - -.. _numpy: https://numpy.org/ - -.. _LAPACK: https://www.netlib.org/lapack/ - -.. _BLAS: https://www.netlib.org/blas/ - -.. _intel: https://www.intel.com/content/www/us/en/developer/articles/technical/numpyscipy-with-intel-mkl.html diff --git a/doc/userguide/usage.rst b/doc/userguide/usage.rst index 3fe5d3a..0829f94 100644 --- a/doc/userguide/usage.rst +++ b/doc/userguide/usage.rst @@ -12,10 +12,6 @@ Importing the solver class The solver classes are all called `Eof` and are stored in interface specific modules: `eofs..Eof`. To import the standard interface solver class:: from eofs.standard import Eof - -the cdms interface solver class:: - - from eofs.cdms import Eof the iris interface solver class:: @@ -33,7 +29,7 @@ Creating an instance of a solver calss can be as simple as:: solver = Eof(data) -but all interfaces have options available. It is highly likely that input data will need to be weighted prior to EOF analysis, which is best done using the *weights* keyword argument. This argument accepts an array of weights in all interfaces, and also a selection of pre-defined weighting method names in the cdms and iris interfaces:: +but all interfaces have options available. It is highly likely that input data will need to be weighted prior to EOF analysis, which is best done using the *weights* keyword argument. This argument accepts an array of weights in all interfaces, and also a selection of pre-defined weighting method names in the iris interface:: solver = Eof(data, weights=weights_array) diff --git a/examples/cdms/hgt_example.py b/examples/cdms/hgt_example.py deleted file mode 100644 index 912acab..0000000 --- a/examples/cdms/hgt_example.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Compute and plot the leading EOF of geopotential height on the 500 hPa -pressure surface over the European/Atlantic sector during winter time. - -This example uses the metadata-retaining cdms2 interface. - - -Additional requirements for this example: - - * cdms2 (http://uvcdat.llnl.gov/) - * matplotlib (http://matplotlib.org/) - * cartopy (http://scitools.org.uk/cartopy/) - -""" -import cartopy.crs as ccrs -import cdms2 -import cdutil -import matplotlib.pyplot as plt -import numpy as np - -from eofs.cdms import Eof -from eofs.examples import example_data_path - - -# Read geopotential height data using the cdms2 module from UV-CDAT. The file -# contains December-February averages of geopotential height at 500 hPa for -# the European/Atlantic domain (80W-40E, 20-90N). -filename = example_data_path('hgt_djf.nc') -ncin = cdms2.open(filename, 'r') -z_djf = ncin('z') -ncin.close() - -# Compute anomalies by removing the time-mean. -z_djf_mean = cdutil.averager(z_djf, axis='t') -z_djf = z_djf - z_djf_mean -z_djf.id = 'z' - -# Create an EOF solver to do the EOF analysis. Square-root of cosine of -# latitude weights are applied before the computation of EOFs. -solver = Eof(z_djf, weights='coslat') - -# Retrieve the leading EOF, expressed as the covariance between the leading PC -# time series and the input SLP anomalies at each grid point. -eof1 = solver.eofsAsCovariance(neofs=1) - -# Plot the leading EOF expressed as covariance in the European/Atlantic domain. -clevs = np.linspace(-75, 75, 11) -lons, lats = eof1.getLongitude()[:], eof1.getLatitude()[:] -proj = ccrs.Orthographic(central_longitude=-20, central_latitude=60) -ax = plt.axes(projection=proj) -ax.set_global() -ax.coastlines() -ax.contourf(lons, lats, eof1(squeeze=True).data, levels=clevs, - cmap=plt.cm.RdBu_r, transform=ccrs.PlateCarree()) -plt.title('EOF1 expressed as covariance', fontsize=16) - -plt.show() diff --git a/examples/cdms/sst_example.py b/examples/cdms/sst_example.py deleted file mode 100644 index d02d75d..0000000 --- a/examples/cdms/sst_example.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -Compute and plot the leading EOF of sea surface temperature in the -central and northern Pacific during winter time. - -The spatial pattern of this EOF is the canonical El Nino pattern, and -the associated time series shows large peaks and troughs for well-known -El Nino and La Nina events. - -This example uses the metadata-retaining cdms2 interface. - -Additional requirements for this example: - - * cdms2 (http://uvcdat.llnl.gov/) - * matplotlib (http://matplotlib.org/) - * cartopy (http://scitools.org.uk/cartopy/) - -""" -import cartopy.crs as ccrs -import cartopy.feature as cfeature -import cdms2 -import matplotlib.pyplot as plt -import numpy as np - -from eofs.cdms import Eof -from eofs.examples import example_data_path - - -# Read SST anomalies using the cdms2 module from UV-CDAT. The file contains -# November-March averages of SST anomaly in the central and northern Pacific. -filename = example_data_path('sst_ndjfm_anom.nc') -ncin = cdms2.open(filename, 'r') -sst = ncin('sst') -ncin.close() - -# Create an EOF solver to do the EOF analysis. Square-root of cosine of -# latitude weights are applied before the computation of EOFs. -solver = Eof(sst, weights='coslat') - -# Retrieve the leading EOF, expressed as the correlation between the leading -# PC time series and the input SST anomalies at each grid point, and the -# leading PC time series itself. -eof1 = solver.eofsAsCorrelation(neofs=1) -pc1 = solver.pcs(npcs=1, pcscaling=1) - -# Plot the leading EOF expressed as correlation in the Pacific domain. -lons, lats = eof1.getLongitude()[:], eof1.getLatitude()[:] -clevs = np.linspace(-1, 1, 11) -ax = plt.axes(projection=ccrs.PlateCarree(central_longitude=190)) -fill = ax.contourf(lons, lats, eof1(squeeze=True).data, clevs, - transform=ccrs.PlateCarree(), cmap=plt.cm.RdBu_r) -ax.add_feature(cfeature.LAND, facecolor='w', edgecolor='k') -cb = plt.colorbar(fill, orientation='horizontal') -cb.set_label('correlation coefficient', fontsize=12) -plt.title('EOF1 expressed as correlation', fontsize=16) - -# Plot the leading PC time series. -plt.figure() -years = range(1962, 2012) -plt.plot(years, pc1.data, color='b', linewidth=2) -plt.axhline(0, color='k') -plt.title('PC1 Time Series') -plt.xlabel('Year') -plt.ylabel('Normalized Units') -plt.xlim(1962, 2012) -plt.ylim(-3, 3) - -plt.show() diff --git a/lib/eofs/__init__.py b/lib/eofs/__init__.py index 64daf20..3cc1d47 100644 --- a/lib/eofs/__init__.py +++ b/lib/eofs/__init__.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - from . import standard from . import tools @@ -29,12 +27,6 @@ # Define the objects imported by imports of the form: from eofs import * __all__ = ['standard', 'tools'] -try: - from . import cdms - __all__.append('cdms') -except ImportError: - pass - try: from . import iris __all__.append('iris') diff --git a/lib/eofs/cdms.py b/lib/eofs/cdms.py deleted file mode 100644 index 73de140..0000000 --- a/lib/eofs/cdms.py +++ /dev/null @@ -1,674 +0,0 @@ -"""Meta-data preserving EOF analysis for `cdms2`.""" -# (c) Copyright 2010-2015 Andrew Dawson. All Rights Reserved. -# -# This file is part of eofs. -# -# eofs is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# eofs is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License -# along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - -import collections.abc - -import cdms2 - -from . import standard -from .tools.cdms import weights_array, cdms2_name - - -class Eof(object): - """EOF analysis (meta-data enabled `cdms2` interface)""" - - def __init__(self, dataset, weights=None, center=True, ddof=1): - """Create an Eof object. - - The EOF solution is computed at initialization time. Method - calls are used to retrieve computed quantities. - - **Argument:** - - *dataset* - A `cdms2` variable containing the data to be analysed. Time - must be the first dimension. Missing values are allowed - provided that they are constant with time (e.g., values of - an oceanographic field over land). - - **Optional arguments:** - - *weights* - Sets the weighting method. The following pre-defined - weighting methods are available: - - * *'area'* : Square-root of grid cell area normalized by - total grid area. Requires a latitude-longitude grid to be - present in the `cdms2` variable *dataset*. This is a - fairly standard weighting strategy. If you are unsure - which method to use and you have gridded data then this - should be your first choice. - - * *'coslat'* : Square-root of cosine of latitude. Requires a - latitude dimension to be present in the `cdms2` variable - *dataset*. - - * *None* : Equal weights for all grid points (*'none'* is - also accepted). - - Alternatively an array of weights whose shape is compatible - with the `cdms2` variable *dataset* may be supplied instead - of specifying a weighting method. - - *center* - If *True*, the mean along the first axis of *dataset* (the - time-mean) will be removed prior to analysis. If *False*, - the mean along the first axis will not be removed. Defaults - to *True* (mean is removed). - - The covariance interpretation relies on the input data being - anomalies with a time-mean of 0. Therefore this option - should usually be set to *True*. Setting this option to - *True* has the useful side effect of propagating missing - values along the time dimension, ensuring that a solution - can be found even if missing values occur in different - locations at different times. - - *ddof* - 'Delta degrees of freedom'. The divisor used to normalize - the covariance matrix is *N - ddof* where *N* is the - number of samples. Defaults to *1*. - - **Returns:** - - *solver* - An `Eof` instance. - - **Examples:** - - EOF analysis with grid-cell-area weighting for the input field:: - - from eofs.cdms import Eof - solver = Eof(dataset, weights='area') - - """ - # Check that dataset is recognised by cdms2 as a variable. - if not cdms2.isVariable(dataset): - raise TypeError('the input data must be a cdms2 variable') - # Store the time axis as an instance variable. - self._timeax = dataset.getTime() - # Verify that a time axis was found, getTime returns None when a - # time axis is not found. - if self._timeax is None: - raise ValueError('time axis not found') - # Check the dimension order of the input, time must be the first - # dimension. - order = dataset.getOrder() - if order[0] != 't': - raise ValueError('time must be the first dimension, ' - 'consider using the reorder() method') - # Verify the presence of at least one spatial dimension. The - # instance variable channels will also be used as a partial axis - # list when constructing meta-data. It contains the spatial - # dimensions. - self._channels = dataset.getAxisList() - self._channels.remove(self._timeax) - if len(self._channels) < 1: - raise ValueError('one or more spatial dimensions are required') - # Store the missing value attribute of the data set in an - # instance variable so that it is recoverable later. - self._missing_value = dataset.getMissing() - # Generate an appropriate set of weights for the input dataset. There - # are several weighting schemes. The 'area' weighting scheme requires - # a latitude-longitude grid to be present, the 'cos_lat' scheme only - # requires a latitude dimension. - if weights is None or (isinstance(weights, str) and weights == 'none'): - # No weights requested, set the weight array to None. - wtarray = None - else: - try: - # Generate a weights array of the appropriate kind, with a - # shape compatible with the data set. - scheme = weights.lower() - wtarray = weights_array(dataset, scheme=scheme) - except AttributeError: - # Weights is not a string, assume it is an array. - wtarray = weights - except ValueError as err: - # Weights is not recognized, raise an error. - raise ValueError(err) - # Cast the wtarray to the same type as the dataset. This prevents the - # promotion of 32-bit input to 64-bit on multiplication with the - # weight array when not required. This will fail with a AttributeError - # exception if the weights array is None, which it may be if no - # weighting was requested. - try: - wtarray = wtarray.astype(dataset.dtype) - except AttributeError: - pass - # Create an EofSolver object using appropriate arguments for this - # data set. The object will be used for the decomposition and - # for returning the results. - self._solver = standard.Eof(dataset.asma(), - weights=wtarray, - center=center, - ddof=ddof) - #: Number of EOFs in the solution. - self.neofs = self._solver.neofs - # name for the dataset. - self._dataset_name = cdms2_name(dataset).replace(' ', '_') - self._dataset_id = dataset.id - - def pcs(self, pcscaling=0, npcs=None): - """Principal component time series (PCs). - - **Optional arguments:** - - *pcscaling* - Set the scaling of the retrieved PCs. The following - values are accepted: - - * *0* : Un-scaled principal components (default). - * *1* : Principal components are scaled to unit variance - (divided by the square-root of their eigenvalue). - * *2* : Principal components are multiplied by the - square-root of their eigenvalue. - - *npcs* - Number of PCs to retrieve. Defaults to all the PCs. If the - number of requested PCs is more than the number that are - available, then all available PCs will be returned. - - **Returns:** - - *pcs* - A `cdms2` variable containing the ordered PCs. The PCs are - numbered from 0 to *npcs* - 1. - - **Examples:** - - All un-scaled PCs:: - - pcs = solver.pcs() - - First 3 PCs scaled to unit variance:: - - pcs = solver.pcs(npcs=3, pcscaling=1) - - """ - pcs = self._solver.pcs(pcscaling, npcs) - pcsax = cdms2.createAxis(list(range(pcs.shape[1])), id='pc') - pcsax.long_name = 'pc_number' - axlist = [self._timeax, pcsax] - pcs = cdms2.createVariable(pcs, id='pcs', axes=axlist) - pcs.long_name = 'principal_components' - return pcs - - def eofs(self, eofscaling=0, neofs=None): - """Emipirical orthogonal functions (EOFs). - - **Optional arguments:** - - *eofscaling* - Sets the scaling of the EOFs. The following values are - accepted: - - * *0* : Un-scaled EOFs (default). - * *1* : EOFs are divided by the square-root of their - eigenvalues. - * *2* : EOFs are multiplied by the square-root of their - eigenvalues. - - *neofs* - Number of EOFs to return. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then all available EOFs will be returned. - - **Returns:** - - *eofs* - A `cdms2` variable containing the ordered EOFs. The EOFs are - numbered from 0 to *neofs* - 1. - - **Examples:** - - All EOFs with no scaling:: - - eofs = solver.eofs() - - First 3 EOFs with scaling applied:: - - eofs = solver.eofs(neofs=3, eofscaling=1) - - """ - eofs = self._solver.eofs(eofscaling, neofs) - eofs.fill_value = self._missing_value - eofax = cdms2.createAxis(list(range(len(eofs))), id='eof') - eofax.long_name = 'eof_number' - axlist = [eofax] + self._channels - eofs = cdms2.createVariable(eofs, - id='eofs', - axes=axlist, - fill_value=self._missing_value) - eofs.long_name = 'empirical_orthogonal_functions' - return eofs - - def eofsAsCorrelation(self, neofs=None): - """ - Empirical orthogonal functions (EOFs) expressed as the - correlation between the principal component time series (PCs) - and the time series of the `Eof` input *dataset* at each grid - point. - - .. note:: - - These are not related to the EOFs computed from the - correlation matrix. - - **Optional argument:** - - *neofs* - Number of EOFs to return. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then all available EOFs will be returned. - - **Returns:** - - *eofs* - A `cdms2` variable containing the ordered EOFs. The EOFs are - numbered from 0 to *neofs* - 1. - - **Examples:** - - All EOFs:: - - eofs = solver.eofsAsCorrelation() - - The leading EOF:: - - eof1 = solver.eofsAsCorrelation(neofs=1) - - """ - eofs = self._solver.eofsAsCorrelation(neofs) - eofs.fill_value = self._missing_value - eofax = cdms2.createAxis(list(range(len(eofs))), id='eof') - eofax.long_name = 'eof_number' - axlist = [eofax] + self._channels - eofs = cdms2.createVariable(eofs, - id='eofs', - axes=axlist, - fill_value=self._missing_value) - eofs.long_name = 'correlation_between_pcs_and_{:s}'.format( - self._dataset_name) - return eofs - - def eofsAsCovariance(self, neofs=None, pcscaling=1): - """ - Empirical orthogonal functions (EOFs) expressed as the - covariance between the principal component time series (PCs) - and the time series of the `Eof` input *dataset* at each grid - point. - - **Optional arguments:** - - *neofs* - Number of EOFs to return. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then all available EOFs will be returned. - - *pcscaling* - Set the scaling of the PCs used to compute covariance. The - following values are accepted: - - * *0* : Un-scaled PCs. - * *1* : PCs are scaled to unit variance (divided by the - square-root of their eigenvalue) (default). - * *2* : PCs are multiplied by the square-root of their - eigenvalue. - - The default is to divide PCs by the square-root of their - eigenvalue so that the PCs are scaled to unit variance - (option 1). - - **Returns:** - - *eofs* - A `cdms2` variable containing the ordered EOFs. The EOFs are - numbered from 0 to *neofs* - 1. - - **Examples:** - - All EOFs:: - - eofs = solver.eofsAsCovariance() - - The leading EOF:: - - eof1 = solver.eofsAsCovariance(neofs=1) - - The leading EOF using un-scaled PCs:: - - eof1 = solver.eofsAsCovariance(neofs=1, pcscaling=0) - - """ - eofs = self._solver.eofsAsCovariance(neofs, pcscaling) - eofs.fill_value = self._missing_value - eofax = cdms2.createAxis(list(range(len(eofs))), id='eof') - axlist = [eofax] + self._channels - eofs = cdms2.createVariable(eofs, - id='eofs_cov', - axes=axlist, - fill_value=self._missing_value) - eofs.long_name = 'covariance_between_pcs_and_{:s}'.format( - self._dataset_name) - return eofs - - def eigenvalues(self, neigs=None): - """Eigenvalues (decreasing variances) associated with each EOF. - - **Optional argument:** - - *neigs* - Number of eigenvalues to return. Defaults to all - eigenvalues.If the number of eigenvalues requested is more - than the number that are available, then all available - eigenvalues will be returned. - - **Returns:** - - *eigenvalues* - A `cdms2` variable containing the eigenvalues arranged - largest to smallest. - - **Examples:** - - All eigenvalues:: - - eigenvalues = solver.eigenvalues() - - The first eigenvalue:: - - eigenvalue1 = solver.eigenvalues(neigs=1) - - """ - lambdas = self._solver.eigenvalues(neigs=neigs) - eofax = cdms2.createAxis(list(range(len(lambdas))), id='eigenvalue') - eofax.long_name = 'eigenvalue_number' - axlist = [eofax] - lambdas = cdms2.createVariable(lambdas, id='eigenvalues', axes=axlist) - lambdas.long_name = 'eigenvalues' - return lambdas - - def varianceFraction(self, neigs=None): - """Fractional EOF variances. - - The fraction of the total variance explained by each EOF mode, - values between 0 and 1 inclusive. - - **Optional argument:** - - *neigs* - Number of eigenvalues to return the fractional variance for. - Defaults to all eigenvalues. If the number of eigenvalues - requested is more than the number that are available, then - fractional variances for all available eigenvalues will be - returned. - - **Returns:** - - *variance_fractions* - A `cdms2` variable containing the fractional variances for - each eigenvalue. The eigenvalues are numbered from 0 to - *neigs* - 1. - - **Examples:** - - The fractional variance represented by each eigenvalue:: - - variance_fractions = solver.varianceFraction() - - The fractional variance represented by the first 3 eigenvalues:: - - variance_fractions = solver.VarianceFraction(neigs=3) - - """ - vfrac = self._solver.varianceFraction(neigs=neigs) - eofax = cdms2.createAxis(list(range(len(vfrac))), id='eigenvalue') - eofax.long_name = 'eigenvalue_number' - axlist = [eofax] - vfrac = cdms2.createVariable(vfrac, id='variance_fractions', - axes=axlist) - vfrac.long_name = 'variance_fractions' - return vfrac - - def totalAnomalyVariance(self): - """ - Total variance associated with the field of anomalies (the sum - of the eigenvalues). - - **Returns:** - - *total_variance* - A scalar value (not a `cdms2` variable). - - **Example:** - - Get the total variance:: - - total_variance = solver.totalAnomalyVariance() - - """ - return self._solver.totalAnomalyVariance() - - def northTest(self, neigs=None, vfscaled=False): - """Typical errors for eigenvalues. - - The method of North et al. (1982) is used to compute the typical - error for each eigenvalue. It is assumed that the number of - times in the input data set is the same as the number of - independent realizations. If this assumption is not valid then - the result may be inappropriate. - - **Optional arguments:** - - *neigs* - The number of eigenvalues to return typical errors for. - Defaults to typical errors for all eigenvalues. - - *vfscaled* - If *True* scale the errors by the sum of the eigenvalues. - This yields typical errors with the same scale as the values - returned by `Eof.varianceFraction`. If *False* then no - scaling is done. Defaults to *False* (no scaling). - - **Returns:** - - *errors* - A `cdms2` variable containing the typical errors for each - eigenvalue. The egienvalues are numbered from 0 to - *neigs* - 1. - - **References** - - North G.R., T.L. Bell, R.F. Cahalan, and F.J. Moeng (1982) - Sampling errors in the estimation of empirical orthogonal - functions. *Mon. Weather. Rev.*, **110**, pp 669-706. - - **Examples:** - - Typical errors for all eigenvalues:: - - errors = solver.northTest() - - Typical errors for the first 3 eigenvalues scaled by the sum of - the eigenvalues:: - - errors = solver.northTest(neigs=3, vfscaled=True) - - """ - typerrs = self._solver.northTest(neigs=neigs, vfscaled=vfscaled) - eofax = cdms2.createAxis(list(range(len(typerrs))), id='eigenvalue') - eofax.long_name = 'eigenvalue_number' - axlist = [eofax] - typerrs = cdms2.createVariable(typerrs, - id='typical_errors', - axes=axlist) - typerrs.long_name = 'typical_errors' - return typerrs - - def reconstructedField(self, neofs): - """Reconstructed data field based on a subset of EOFs. - - If weights were passed to the `Eof` instance the returned - reconstructed field will automatically have this weighting - removed. Otherwise the returned field will have the same - weighting as the `Eof` input *dataset*. - - **Argument:** - - *neofs* - Number of EOFs to use for the reconstruction. - Alternatively this argument can be an iterable of mode - numbers (where the first mode is 1) in order to facilitate - reconstruction with arbitrary modes. - - **Returns:** - - *reconstruction* - A `cdms2` variable with the same dimensions `Eof` input - *dataset* containing the reconstruction using *neofs* EOFs. - - **Example:** - - Reconstruct the input field using 3 EOFs:: - - reconstruction = solver.reconstructedField(3) - - Reconstruct the input field using EOFs 1, 2 and 5:: - - reconstruction = solver.reconstuctedField([1, 2, 5]) - - """ - rfield = self._solver.reconstructedField(neofs) - rfield.fill_value = self._missing_value - axlist = [self._timeax] + self._channels - rfield = cdms2.createVariable(rfield, - id=self._dataset_id, - axes=axlist, - fill_value=self._missing_value) - if isinstance(neofs, collections.abc.Iterable): - name_part = 'EOFs_{}'.format('_'.join([str(e) for e in neofs])) - else: - name_part = '{:d}_EOFs'.format(neofs) - rfield.long_name = '{:s}_reconstructed_with_{:s}'.format( - self._dataset_name, name_part) - rfield.neofs = neofs - return rfield - - def projectField(self, field, neofs=None, eofscaling=0, weighted=True): - """Project a field onto the EOFs. - - Given a data set, projects it onto the EOFs to generate a - corresponding set of pseudo-PCs. - - **Argument:** - - *field* - A `cdms2` variable containing the field to project onto the - EOFs. It must have the same corresponding spatial dimensions - (including missing values in the same places) as the `Eof` - input *dataset*. It may have a different length time - dimension to the `Eof` input *dataset* or no time dimension - at all. If a time dimension exists it must be the first - dimension. - - **Optional arguments:** - - *neofs* - Number of EOFs to project onto. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then the field will be projected onto all - available EOFs. - - *eofscaling* - Set the scaling of the EOFs that are projected - onto. The following values are accepted: - - * *0* : Un-scaled EOFs (default). - * *1* : EOFs are divided by the square-root of their eigenvalue. - * *2* : EOFs are multiplied by the square-root of their - eigenvalue. - - *weighted* - If *True* then the field is weighted using the same weights - used for the EOF analysis prior to projection. If *False* - then no weighting is applied. Defaults to *True* (weighting - is applied). Generally only the default setting should be - used. - - **Returns:** - - *pseudo_pcs* - A `cdms2` variable containing the pseudo-PCs. The PCs are - numbered from 0 to *neofs* - 1. - - **Examples:** - - Project a field onto all EOFs:: - - pseudo_pcs = solver.projectField(field) - - Project fields onto the three leading EOFs:: - - pseudo_pcs = solver.projectField(field, neofs=3) - - """ - # Check that field is recognised by cdms2 as a variable. - if not cdms2.isVariable(field): - raise TypeError('the input field must be a cdms2 variable') - dataset_name = cdms2_name(field).replace(' ', '_') - # Compute the projected PCs. - pcs = self._solver.projectField(field.asma(), - neofs=neofs, - eofscaling=eofscaling, - weighted=weighted) - # Construct the required axes. - if pcs.ndim == 2: - # 2D PCs require a time axis and a PC axis. - pcsax = cdms2.createAxis(list(range(pcs.shape[1])), id='pc') - pcsax.long_name = 'pc_number' - timeax = field.getAxis(0) # time is assumed to be first anyway - axlist = [timeax, pcsax] - else: - # 1D PCs require only a PC axis. - pcsax = cdms2.createAxis(list(range(pcs.shape[0])), id='pc') - pcsax.long_name = 'pc_number' - axlist = [pcsax] - # Apply meta data to the projected PCs. - pcs = cdms2.createVariable(pcs, id='pseudo_pcs', axes=axlist) - pcs.long_name = '{:s}_pseudo_pcs'.format(dataset_name) - return pcs - - def getWeights(self): - """Weights used for the analysis. - - **Returns:** - - *weights* - An array contaning the analysis weights (not a `cdms2` - variable). - - **Example:** - - The weights used for the analysis:: - - weights = solver.getWeights() - - """ - return self._solver.getWeights() diff --git a/lib/eofs/examples/__init__.py b/lib/eofs/examples/__init__.py index 4a53a5b..265ebb3 100644 --- a/lib/eofs/examples/__init__.py +++ b/lib/eofs/examples/__init__.py @@ -15,7 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa import os diff --git a/lib/eofs/iris.py b/lib/eofs/iris.py index 13bdd4f..4f6c042 100644 --- a/lib/eofs/iris.py +++ b/lib/eofs/iris.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import collections.abc from copy import copy @@ -27,7 +25,7 @@ from .tools.iris import get_time_coord, weights_array, classified_aux_coords -class Eof(object): +class Eof: """EOF analysis (meta-data enabled `iris` interface)""" def __init__(self, cube, weights=None, center=True, ddof=1): diff --git a/lib/eofs/multivariate/__init__.py b/lib/eofs/multivariate/__init__.py index 6d14376..78587e2 100644 --- a/lib/eofs/multivariate/__init__.py +++ b/lib/eofs/multivariate/__init__.py @@ -15,20 +15,12 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - from . import standard __all__ = ['standard'] -try: - from . import cdms - __all__.append('cdms') -except ImportError: - pass - try: from . import iris __all__.append('iris') diff --git a/lib/eofs/multivariate/cdms.py b/lib/eofs/multivariate/cdms.py deleted file mode 100644 index 2902f17..0000000 --- a/lib/eofs/multivariate/cdms.py +++ /dev/null @@ -1,693 +0,0 @@ -"""Multivariate EOF analysis for `cdms2` variables.""" -# (c) Copyright 2013-2015 Andrew Dawson. All Rights Reserved. -# -# This file is part of eofs. -# -# eofs is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# eofs is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License -# along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - -import collections.abc - -import cdms2 - -from eofs.tools.cdms import weights_array, cdms2_name -from . import standard - - -class MultivariateEof(object): - """Multivariate EOF analysis (meta-data enabled `cdms2` interface)""" - - def __init__(self, datasets, weights=None, center=True, ddof=1): - """Create a MultivariateEof object. - - The EOF solution is computed at initialization time. Method - calls are used to retrieve computed quantities. - - **Arguments:** - - *datasets* - A list/tuple containing one or more `cdms2` variables, each - two or more dimensions, containing the data to be analysed. - Time must be the first dimension of each variable. Missing - values are allowed provided that they are constant with time - in each field (e.g., values of an oceanographic field over - land). - - **Optional arguments:** - - *weights* - Sets the weighting method. One method can be chosen to apply - to all variables in *datasets* or a sequence of options can - be given to specify a different weighting method for each - variable in *datasets*. The following pre-defined weighting - methods are available: - - * *'area'* : Square-root of grid cell area normalized by - total grid area. Requires a latitude-longitude grid to be - present in the corresponding `cdms2` variable. This is a - fairly standard weighting strategy. If you are unsure - which method to use and you have gridded data then this - should be your first choice. - - * *'coslat'* : Square-root of cosine of latitude. Requires a - latitude dimension to be present in the corresponding - `cdms2` variable. - - * *None* : Equal weights for all grid points (*'none'* is - also accepted). - - Alternatively a sequence of arrays of weights whose shapes - are compatible with the corresponding `cdms2` variables in - *datasets* may be supplied instead of specifying a - weighting method. - - *center* - If *True*, the mean along the first axis of each variable in - *datasets* (the time-mean) will be removed prior to - analysis. If *False*, the mean along the first axis will not - be removed. Defaults to *True* (mean is removed). - - The covariance interpretation relies on the input data being - anomalies with a time-mean of 0. Therefore this option - should usually be set to *True*. Setting this option to - *True* has the useful side effect of propagating missing - values along the time dimension, ensuring that a solution - can be found even if missing values occur in different - locations at different times. - - *ddof* - 'Delta degrees of freedom'. The divisor used to normalize - the covariance matrix is *N - ddof* where *N* is the - number of samples. Defaults to *1*. - - **Returns:** - - *solver* - An `MultivariateEof` instance. - - **Examples:** - - EOF analysis with grid-cell-area weighting using two input - fields:: - - from eofs.multivariate.cdms import MultivariateEof - solver = MultivariateEof([var1, var2], weights='area') - - """ - # Record the number of datasets. - self._ndata = len(datasets) - # Ensure the weights are specified one per dataset. - if weights in ('none', None, 'area', 'coslat'): - weights = [weights] * self._ndata - elif len(weights) != self._ndata: - raise ValueError('number of weights is incorrect, ' - 'expecting {:d} but got {:d}'.format( - self._ndata, len(weights))) - # Record dimension information, missing values and compute weights. - self._multitimeaxes = list() - self._multichannels = list() - self._multimissing = list() - passweights = list() - for dataset, weight in zip(datasets, weights): - if not cdms2.isVariable(dataset): - raise TypeError('the input data set must be a cdms2 variable') - # Ensure a time dimension exists. - timeaxis = dataset.getTime() - if timeaxis is None: - raise ValueError('time axis not found') - self._multitimeaxes.append(timeaxis) - # Ensure the time dimension is the first dimension. - order = dataset.getOrder() - if order[0] != "t": - raise ValueError('time must be the first dimension, ' - 'consider using the reorder() method') - # Record the other dimensions. - channels = dataset.getAxisList() - channels.remove(timeaxis) - if len(channels) < 1: - raise ValueError('one or more spatial dimensions are required') - self._multichannels.append(channels) - # Record the missing values. - self._multimissing.append(dataset.getMissing()) - # Compute weights as required. - if weight in ("none", None): - passweights.append(None) - else: - try: - wtarray = weights_array(dataset, scheme=weight.lower()) - passweights.append(wtarray) - except AttributeError: - # Weight specification is not a string. Assume it is an - # array of weights. - passweights.append(weight) - # any other error will be raised - # Define a time axis as the time axis of the first dataset. - self._timeax = self._multitimeaxes[0] - # Create a MultipleEofSolver to do the computations. - self._solver = standard.MultivariateEof([d.asma() for d in datasets], - weights=passweights, - center=center, - ddof=ddof) - #: Number of EOFs in the solution. - self.neofs = self._solver.neofs - # Names of the input variables. - self._dataset_names = [cdms2_name(v).replace(' ', '_') - for v in datasets] - self._dataset_ids = [dataset.id for dataset in datasets] - - def pcs(self, pcscaling=0, npcs=None): - """Principal component time series (PCs). - - **Optional arguments:** - - *pcscaling* - Set the scaling of the retrieved PCs. The following - values are accepted: - - * *0* : Un-scaled PCs (default). - * *1* : PCs are scaled to unit variance (divided by the - square-root of their eigenvalue). - * *2* : PCs are multiplied by the square-root of their - eigenvalue. - - *npcs* - Number of PCs to retrieve. Defaults to all the PCs. If the - number of PCs requested is more than the number that are - available, then all available PCs will be returned. - - **Returns:** - - *pcs* - A `cdms2` variable containing the ordered PCs. - - **Examples:** - - All un-scaled PCs:: - - pcs = solver.pcs() - - First 3 PCs scaled to unit variance:: - - pcs = solver.pcs(npcs=3, pcscaling=1) - - """ - pcs = self._solver.pcs(pcscaling, npcs) - pcsax = cdms2.createAxis(list(range(pcs.shape[1])), id='pc') - pcsax.long_name = 'pc_number' - axlist = [self._timeax, pcsax] - pcs = cdms2.createVariable(pcs, id='pcs', axes=axlist) - pcs.long_name = 'principal_components' - return pcs - - def eofs(self, eofscaling=0, neofs=None): - """Empirical orthogonal functions (EOFs). - - **Optional arguments:** - - *eofscaling* - Sets the scaling of the EOFs. The following values are - accepted: - - * *0* : Un-scaled EOFs (default). - * *1* : EOFs are divided by the square-root of their - eigenvalues. - * *2* : EOFs are multiplied by the square-root of their - eigenvalues. - - *neofs* - Number of EOFs to return. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then all available EOFs will be returned. - - **Returns:** - - *eofs_list* - A list of `cdms2` variables containing the ordered EOFs for - each variable. - - **Examples:** - - All EOFs with no scaling:: - - eofs_list = solver.eofs() - - The leading EOF with scaling applied:: - - eof1_list = solver.eofs(neofs=1, eofscaling=1) - - """ - eofset = self._solver.eofs(eofscaling, neofs) - neofs = eofset[0].shape[0] - eofax = cdms2.createAxis(list(range(neofs)), id='eof') - eofax.long_name = 'eof_number' - for iset in range(self._ndata): - axlist = [eofax] + self._multichannels[iset] - eofset[iset].fill_value = self._multimissing[iset] - eofset[iset] = cdms2.createVariable( - eofset[iset], - id='eofs', - axes=axlist, - fill_value=self._multimissing[iset]) - eofset[iset].long_name = 'empirical_orthogonal_functions' - return eofset - - def eofsAsCorrelation(self, neofs=None): - """ - Empirical orthogonal functions (EOFs) expressed as the - correlation between the principal component time series (PCs) - and the each data set in the `MultivariateEof` input *datasets*. - - .. note:: - - These are not related to the EOFs computed from the - correlation matrix. - - **Optional argument:** - - *neofs* - Number of EOFs to return. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then all available EOFs will be returned. - - **Returns:** - - *eofs_list* - A list of `cdms2` variables containing the ordered EOFs for - each variable. - - **Examples:** - - All EOFs of each data set:: - - eofs_list = solver.eofsAsCorrelation() - - The leading EOF of each data set:: - - eof1_list = solver.eofsAsCorrelation(neofs=1) - - """ - eofset = self._solver.eofsAsCorrelation(neofs) - neofs = eofset[0].shape[0] - eofax = cdms2.createAxis(list(range(neofs)), id='eof') - eofax.long_name = 'eof_number' - for iset in range(self._ndata): - axlist = [eofax] + self._multichannels[iset] - eofset[iset].fill_value = self._multimissing[iset] - eofset[iset] = cdms2.createVariable( - eofset[iset], - id='eofs', - axes=axlist, - fill_value=self._multimissing[iset]) - eofset[iset].long_name = 'correlation_between_pcs_and_{:s}'.format( - self._dataset_names[iset]) - return eofset - - def eofsAsCovariance(self, neofs=None, pcscaling=1): - """ - Empirical orthogonal functions (EOFs) expressed as the - covariance between the principal component time series (PCs) - and the each data set in the `MultivariateEof` input *datasets*. - - **Optional argument:** - - *neofs* - Number of EOFs to return. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then all available EOFs will be returned. - - *pcscaling* - Set the scaling of the PCs used to compute covariance. The - following values are accepted: - - * *0* : Un-scaled PCs. - * *1* : PCs are scaled to unit variance (divided by the - square-root of their eigenvalue) (default). - * *2* : PCs are multiplied by the square-root of their - eigenvalue. - - The default is to divide PCs by the square-root of their - eigenvalue so that the PCs are scaled to unit variance - (option 1). - - **Returns:** - - *eofs_list* - A list of `cdms2` variables containing the ordered EOFs for - each variable. - - **Examples:** - - All EOFs of each data set:: - - eofs_list = solver.eofsAsCovariance() - - The leading EOF of each data set:: - - eof1_list = solver.eofsAsCovariance(neofs=1) - - """ - eofset = self._solver.eofsAsCovariance(neofs) - neofs = eofset[0].shape[0] - eofax = cdms2.createAxis(list(range(neofs)), id='eof') - eofax.long_name = 'eof_number' - for iset in range(self._ndata): - axlist = [eofax] + self._multichannels[iset] - eofset[iset].fill_value = self._multimissing[iset] - eofset[iset] = cdms2.createVariable( - eofset[iset], - id='eofs', - axes=axlist, - fill_value=self._multimissing[iset]) - eofset[iset].long_name = 'covariance_between_pcs_and_{:s}'.format( - self._dataset_names[iset]) - return eofset - - def eigenvalues(self, neigs=None): - """ - Eigenvalues (decreasing variances) associated with each EOF - mode. - - **Optional argument:** - - *neigs* - Number of eigenvalues to return. Defaults to all - eigenvalues. If the number of eigenvalues requested is more - than the number that are available, then all available - eigenvalues will be returned. - - **Returns:** - - *eigenvalues* - A `cdms2` variable containing the eigenvalues arranged - largest to smallest. - - **Examples:** - - All eigenvalues:: - - eigenvalues = solver.eigenvalues() - - The first eigenvalue:: - - eigenvalue1 = solver.eigenvalues(neigs=1) - - """ - lambdas = self._solver.eigenvalues(neigs=neigs) - eofax = cdms2.createAxis(list(range(len(lambdas))), id='eigenvalue') - eofax.long_name = 'eigenvalue_number' - axlist = [eofax] - lambdas = cdms2.createVariable(lambdas, id='eigenvalues', axes=axlist) - lambdas.long_name = 'eigenvalues' - return lambdas - - def varianceFraction(self, neigs=None): - """Fractional EOF mode variances. - - The fraction of the total variance explained by each EOF mode, - values between 0 and 1 inclusive. - - **Optional argument:** - - *neigs* - Number of eigenvalues to return the fractional variance for. - Defaults to all eigenvalues. If the number of eigenvalues - requested is more than the number that are available, then - fractional variances for all available eigenvalues will be - returned. - - **Returns:** - - *variance_fractions* - A `cdms2` variable containing the fractional variances. - - **Examples:** - - The fractional variance represented by each EOF mode:: - - variance_fractions = solver.varianceFraction() - - The fractional variance represented by the first EOF mode:: - - variance_fraction_mode_1 = solver.VarianceFraction(neigs=1) - - """ - vfrac = self._solver.varianceFraction(neigs=neigs) - eofax = cdms2.createAxis(list(range(len(vfrac))), id='eigenvalue') - axlist = [eofax] - vfrac = cdms2.createVariable(vfrac, id='variance_fraction', - axes=axlist) - vfrac.long_name = 'variance_fraction' - return vfrac - - def totalAnomalyVariance(self): - """ - Total variance associated with the field of anomalies (the sum - of the eigenvalues). - - **Returns:** - - *total_variance* - A scalar value (not a `cdms2` variable). - - **Example:** - - Get the total variance:: - - total_variance = solver.totalAnomalyVariance() - - """ - return self._solver.totalAnomalyVariance() - - def northTest(self, neigs=None, vfscaled=False): - """Typical errors for eigenvalues. - - The method of North et al. (1982) is used to compute the typical - error for each eigenvalue. It is assumed that the number of - times in the input data set is the same as the number of - independent realizations. If this assumption is not valid then - the result may be inappropriate. - - **Optional arguments:** - - *neigs* - The number of eigenvalues to return typical errors for. - Defaults to typical errors for all eigenvalues. If the - number of eigenvalues requested is more than the number that - are available, then typical errors for all available - eigenvalues will be returned. - - *vfscaled* - If *True* scale the errors by the sum of the eigenvalues. - This yields typical errors with the same scale as the values - returned by `MultivariateEof.varianceFraction`. If *False* - then no scaling is done. Defaults to *False* (no scaling). - - **Returns:** - - *errors* - A `cdms2` variable containing the typical errors. - - **References** - - North G.R., T.L. Bell, R.F. Cahalan, and F.J. Moeng (1982) - Sampling errors in the estimation of empirical orthogonal - functions. *Mon. Weather. Rev.*, **110**, pp 669-706. - - **Examples:** - - Typical errors for all eigenvalues:: - - errors = solver.northTest() - - Typical errors for the first 5 eigenvalues scaled by the sum of - the eigenvalues:: - - errors = solver.northTest(neigs=5, vfscaled=True) - - """ - typerrs = self._solver.northTest(neigs=neigs, vfscaled=vfscaled) - eofax = cdms2.createAxis(list(range(len(typerrs))), id='eigenvalue') - eofax.long_name = 'eof_number' - axlist = [eofax] - typerrs = cdms2.createVariable(typerrs, - id='typical_errors', - axes=axlist) - typerrs.long_name = 'typical_errors' - return typerrs - - def reconstructedField(self, neofs): - """Reconstructed data sets based on a subset of EOFs. - - If weights were passed to the `MultivariateEof` instance the - returned reconstructed fields will automatically have this - weighting removed. Otherwise each returned field will have the - same weighting as the corresponding array in the - `MultivariateEof` input *datasets*. - - **Argument:** - - *neofs* - Number of EOFs to use for the reconstruction. If the - number of EOFs requested is more than the number that are - available, then all available EOFs will be used for the - reconstruction. Alternatively this argument can be an - iterable of mode numbers (where the first mode is 1) in - order to facilitate reconstruction with arbitrary modes. - - **Returns:** - - *reconstruction_list* - A list of `cdms2` variable with the same dimensions as the - variables in the `MultivariateEof` input *datasets* - contaning the reconstructions using *neofs* EOFs. - - **Example:** - - Reconstruct the input data sets using 3 EOFs:: - - reconstruction_list = solver.reconstructedField(neofs=3) - - Reconstruct the input field using EOFs 1, 2 and 5:: - - reconstruction_list = solver.reconstuctedField([1, 2, 5]) - - """ - rfset = self._solver.reconstructedField(neofs) - if isinstance(neofs, collections.abc.Iterable): - name_part = 'EOFs_{}'.format('_'.join([str(e) for e in neofs])) - else: - name_part = '{:d}_EOFs'.format(neofs) - for iset in range(self._ndata): - axlist = [self._multitimeaxes[iset]] + self._multichannels[iset] - rfset[iset].fill_value = self._multimissing[iset] - rfset[iset] = cdms2.createVariable( - rfset[iset], - id=self._dataset_ids[iset], - axes=axlist, - fill_value=self._multimissing[iset]) - rfset[iset].long_name = '{:s}_reconstructed_with_{:s}'.format( - self._dataset_names[iset], name_part) - rfset[iset].neofs = neofs - return rfset - - def projectField(self, fields, neofs=None, eofscaling=0, weighted=True): - """Project a set of fields onto the EOFs. - - Given a set of fields, projects them onto the EOFs to generate a - corresponding set of pseudo-PCs. - - **Argument:** - - *fields* - A list/tuple containing one or more `cdms2` variables, each - with two or more dimensions, containing the data to be - projected onto the EOFs. Each field must have the same - spatial dimensions (including missing values in the same - places) as the corresponding data set in the - `MultivariateEof` input *datasets*. The fields may have - different length time dimensions to the `MultivariateEof` - inputs *datasets* or no time dimension at all, but this - must be consistent for all fields. - - **Optional arguments:** - - *neofs* - Number of EOFs to project onto. Defaults to all EOFs. If the - number of EOFs requested is more than the number that are - available, then the field will be projected onto all - available EOFs. - - *eofscaling* - Set the scaling of the EOFs that are projected - onto. The following values are accepted: - - * *0* : Un-scaled EOFs (default). - * *1* : EOFs are divided by the square-root of their - eigenvalue. - * *2* : EOFs are multiplied by the square-root of their - eigenvalue. - - *weighted* - If *True* then each field in *fields* is weighted using the - same weights used for the EOF analysis prior to projection. - If *False* then no weighting is applied. Defaults to *True* - (weighting is applied). Generally only the default setting - should be used. - - **Returns:** - - *pseudo_pcs* - A `cdms2` variable containing the ordered pseudo-PCs. - - **Examples:** - - Project a data set onto all EOFs:: - - pseudo_pcs = solver.projectField([field1, field2]) - - Project a data set onto the four leading EOFs:: - - pseudo_pcs = solver.projectField([field1, field2], neofs=4) - - """ - for field in fields: - if not cdms2.isVariable(field): - raise TypeError('the input data set must be a cdms2 variable') - if len(fields) != self._ndata: - raise ValueError('number of fields is incorrect, expecting {:d} ' - 'but got {:d}'.format(self._ndata, len(fields))) - for field in fields: - order = field.getOrder() - if 't' in order: - if order[0] != 't': - raise ValueError('time must be the first dimension, ' - 'consider using the reorder() method') - pcs = self._solver.projectField([f.asma() for f in fields], - neofs=neofs, - eofscaling=eofscaling, - weighted=weighted) - # Create an axis list, its contents depend on whether or not a time - # axis was present in the input field. - if pcs.ndim == 2: - # Time dimension present: - pcsax = cdms2.createAxis(list(range(pcs.shape[1])), id='pc') - pcsax.long_name = 'pc_number' - axlist = [fields[0].getAxis(0), pcsax] - else: - # A PC axis and a leading time axis. - pcsax = cdms2.createAxis(list(range(pcs.shape[0])), id='pc') - pcsax.long_name = 'pc_number' - axlist = [pcsax] - # Apply meta data to the projected PCs. - pcs = cdms2.createVariable(pcs, id='pseudo_pcs', axes=axlist) - pcs.long_name = 'psuedo_pcs' - return pcs - - def getWeights(self): - """Weights used for the analysis. - - **Returns:** - - *weights_list* - A list of arrays containing the analysis weights for each - variable (not `cdms2` variables). - - **Example:** - - The weights used for the analysis:: - - weights_list = solver.getWeights() - - """ - return self._solver.getWeights() diff --git a/lib/eofs/multivariate/iris.py b/lib/eofs/multivariate/iris.py index b88a0f6..cc3564a 100644 --- a/lib/eofs/multivariate/iris.py +++ b/lib/eofs/multivariate/iris.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import collections.abc from copy import copy @@ -28,7 +26,7 @@ from . import standard -class MultivariateEof(object): +class MultivariateEof: """Multivariate EOF analysis (meta-data enabled `iris` interface)""" def __init__(self, cubes, weights=None, center=True, ddof=1): diff --git a/lib/eofs/multivariate/standard.py b/lib/eofs/multivariate/standard.py index 3943110..469d63c 100644 --- a/lib/eofs/multivariate/standard.py +++ b/lib/eofs/multivariate/standard.py @@ -15,14 +15,12 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np from eofs import standard -class MultivariateEof(object): +class MultivariateEof: """Multivariate EOF analysis (`numpy` interface)""" def __init__(self, datasets, weights=None, center=True, ddof=1): diff --git a/lib/eofs/standard.py b/lib/eofs/standard.py index 811b528..89306ee 100644 --- a/lib/eofs/standard.py +++ b/lib/eofs/standard.py @@ -16,7 +16,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa import collections.abc import warnings @@ -32,7 +31,7 @@ from .tools.standard import correlation_map, covariance_map -class Eof(object): +class Eof: """EOF analysis (`numpy` interface)""" def __init__(self, dataset, weights=None, center=True, ddof=1): @@ -121,7 +120,7 @@ def __init__(self, dataset, weights=None, center=True, ddof=1): # Store information about the shape/size of the input data. self._records = self._data.shape[0] self._originalshape = self._data.shape[1:] - channels = np.product(self._originalshape) + channels = np.prod(self._originalshape) # Weight the data set according to weighting argument. if weights is not None: try: @@ -196,7 +195,7 @@ def __init__(self, dataset, weights=None, center=True, ddof=1): # astype method to ensure the eigenvectors are the same type as the # input dataset since multiplication by np.NaN will promote to 64-bit. self._flatE = np.ones([self.neofs, channels], - dtype=self._data.dtype) * np.NaN + dtype=self._data.dtype) * np.nan self._flatE = self._flatE.astype(self._data.dtype) self._flatE[:, nonMissingIndex] = E # Remove the scaling on the principal component time-series that is @@ -738,7 +737,7 @@ def projectField(self, field, neofs=None, eofscaling=0, weighted=True): if eof_ndim > input_ndim: field = field.reshape((1,) + field.shape) records = field.shape[0] - channels = np.product(field.shape[1:]) + channels = np.prod(field.shape[1:]) field_flat = field.reshape([records, channels]) # Locate the non-missing values and isolate them. if not self._valid_nan(field_flat): diff --git a/lib/eofs/tests/__init__.py b/lib/eofs/tests/__init__.py index 24f1830..e9f986e 100644 --- a/lib/eofs/tests/__init__.py +++ b/lib/eofs/tests/__init__.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np import numpy.ma as ma @@ -24,7 +22,7 @@ np.seterr(all='ignore') -class EofsTest(object): +class EofsTest: """Base class for tests.""" def _tomasked(self, value): diff --git a/lib/eofs/tests/reference.py b/lib/eofs/tests/reference.py index 0a45b03..1f87ca0 100644 --- a/lib/eofs/tests/reference.py +++ b/lib/eofs/tests/reference.py @@ -65,63 +65,6 @@ def _read_reference_solution(weights): return fields -def _wrap_cdms(solution, neofs, time_units): - try: - import cdms2 - except ImportError: - raise ValueError("cannot use container 'cdms' without " - "the cdms2 module") - time_dim = cdms2.createAxis(solution['time'], id='time') - time_dim.designateTime() - time_dim.units = time_units - lat_dim = cdms2.createAxis(solution['latitude'], id='latitude') - lat_dim.designateLatitude() - lon_dim = cdms2.createAxis(solution['longitude'], id='longitude') - lon_dim.designateLongitude() - eof_dim = cdms2.createAxis(np.arange(1, neofs+1), id='eof') - eof_dim.long_name = 'eof_number' - solution['sst'] = cdms2.createVariable( - solution['sst'], - axes=[time_dim, lat_dim, lon_dim], - id='sst') - solution['eigenvalues'] = cdms2.createVariable( - solution['eigenvalues'], - axes=[eof_dim], - id='eigenvalues') - solution['eofs'] = cdms2.createVariable( - solution['eofs'], - axes=[eof_dim, lat_dim, lon_dim], - id='eofs') - solution['pcs'] = cdms2.createVariable( - solution['pcs'], - axes=[time_dim, eof_dim], - id='pcs') - solution['variance'] = cdms2.createVariable( - solution['variance'], - axes=[eof_dim], - id='variance') - solution['eofscor'] = cdms2.createVariable( - solution['eofscor'], - axes=[eof_dim, lat_dim, lon_dim], - id='eofscor') - solution['eofscov'] = cdms2.createVariable( - solution['eofscov'], - axes=[eof_dim, lat_dim, lon_dim], - id='eofscov') - solution['errors'] = cdms2.createVariable( - solution['errors'], - axes=[eof_dim], - id='errors') - solution['scaled_errors'] = cdms2.createVariable( - solution['scaled_errors'], - axes=[eof_dim], - id='scaled_errors') - solution['rcon'] = cdms2.createVariable( - solution['rcon'], - axes=[time_dim, lat_dim, lon_dim], - id='reconstructed_sst') - - def _wrap_iris(solution, neofs, time_units): try: from iris.cube import Cube @@ -189,11 +132,9 @@ def _wrap_xarray(solution, neofs, time_units): try: import xarray as xr except ImportError: - try: - import xray as xr - except ImportError: - raise ValueError("cannot use container 'xarray' without " - "the xarray/xray module") + raise ValueError( + "cannot use container 'xarray' without the xarray module" + ) time_dim = xr.IndexVariable('time', solution['time']) lat_dim = xr.IndexVariable('latitude', solution['latitude']) lon_dim = xr.IndexVariable('longitude', solution['longitude']) @@ -208,7 +149,6 @@ def _wrap_xarray(solution, neofs, time_units): def _get_wrapper(container_type): return {'standard': lambda *args: None, - 'cdms': _wrap_cdms, 'iris': _wrap_iris, 'xarray': _wrap_xarray}[container_type] @@ -219,7 +159,7 @@ def reference_solution(container_type, weights): **Arguments:** *container_type* - Either 'standard', 'cdms', 'iris' or 'xarray'. + Either 'standard', 'iris' or 'xarray'. *weights* Weights method. One of 'equal', 'latitude', or 'area'. @@ -227,7 +167,7 @@ def reference_solution(container_type, weights): """ container_type = container_type.lower() weights = weights.lower() - if container_type not in ('standard', 'iris', 'cdms', 'xarray'): + if container_type not in ('standard', 'iris', 'xarray'): raise ValueError("unknown container type " "'{!s}'".format(container_type)) solution = _read_reference_solution(weights) @@ -243,7 +183,7 @@ def reference_multivariate_solution(container_type, weights): **Arguments:** *container_type* - Either 'standard', 'cdms', or 'iris'. + Either 'standard' or 'iris'. *weights* Weights method. One of 'equal', 'latitude', 'area', diff --git a/lib/eofs/tests/test_error_handling.py b/lib/eofs/tests/test_error_handling.py index ca9d7c0..43d6784 100644 --- a/lib/eofs/tests/test_error_handling.py +++ b/lib/eofs/tests/test_error_handling.py @@ -15,13 +15,7 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np -try: - import cdms2 -except ImportError: - pass import pytest import eofs @@ -32,10 +26,6 @@ # Create a mapping from interface name to solver class. solvers = {'standard': eofs.standard.Eof} -try: - solvers['cdms'] = eofs.cdms.Eof -except AttributeError: - pass try: solvers['iris'] = eofs.iris.Eof except AttributeError: @@ -106,22 +96,6 @@ class TestErroHandlersStandard(ErrorHandlersTest): weights = 'equal' -# ---------------------------------------------------------------------------- -# Error Handler tests for the cdms interface - - -class TestErrorHandlersCDMS(ErrorHandlersTest): - """Test error handling in the cdms interface.""" - interface = 'cdms' - weights = 'equal' - - def test_projectField_invalid_type(self): - # projecting a field of the wrong type - solution = reference_solution('standard', 'equal') - with pytest.raises(TypeError): - pcs = self.solver.projectField(solution['sst']) - - # ---------------------------------------------------------------------------- # Error Handler tests for the iris interface @@ -242,85 +216,6 @@ def test_input_with_non_uniform_missing_values(self): solver = self.solver_class(data, center=False) -# ---------------------------------------------------------------------------- -# Constructor tests for the cdms interface - - -class TestConstructorCDMS(EofsTest): - """Test the error handling in the cdms interface constructor.""" - - @classmethod - def setup_class(cls): - try: - cls.solver_class = solvers['cdms'] - except KeyError: - pytest.skip('missing dependencies required to test the ' - 'cdms interface') - - def test_wrong_input_type(self): - # input of the wrong type - solution = reference_solution('standard', 'equal') - data = solution['sst'] - with pytest.raises(TypeError): - solver = self.solver_class(data) - - def test_input_without_time_dimension(self): - # no time dimension in the input - solution = reference_solution('cdms', 'equal') - data = solution['sst'](time=slice(0, 1), squeeze=True) - with pytest.raises(ValueError): - solver = self.solver_class(data) - - def test_input_time_dimension_not_first(self): - # time not the first dimension in the input - solution = reference_solution('cdms', 'equal') - data = solution['sst'] - data = data.reorder('xyt') - with pytest.raises(ValueError): - solver = self.solver_class(data) - - def test_input_no_spatial_dimensions(self): - # not enough dimensions in the input - solution = reference_solution('cdms', 'equal') - data = solution['sst'][:, 0, 0] - with pytest.raises(ValueError): - solver = self.solver_class(data) - - def test_invalid_builtin_weights_value(self): - # invalid weighting scheme name - solution = reference_solution('cdms', 'equal') - data = solution['sst'] - with pytest.raises(ValueError): - solver = self.solver_class(data, weights='invalid') - - def test_builtin_latitude_weights_with_missing_dimension(self): - # latitude weights without latitude dimension - solution = reference_solution('cdms', 'equal') - data = solution['sst'](latitude=slice(0, 1), squeeze=True) - with pytest.raises(ValueError): - solver = self.solver_class(data, weights='coslat') - - def test_builtin_area_weights_with_missing_dimension(self): - # area weights without longitude dimension - solution = reference_solution('cdms', 'equal') - data = solution['sst'](longitude=slice(0, 1), squeeze=True) - with pytest.raises(ValueError): - solver = self.solver_class(data, weights='area') - - def test_builtin_area_weights_with_non_adjacent_dimensions(self): - # area weights with latitude and longitude not adjacent in input - solution = reference_solution('cdms', 'equal') - data = solution['sst'] - newdim = cdms2.createAxis([0.], id='height') - newdim.designateLevel() - data = cdms2.createVariable(cdms2.MV.reshape(data, data.shape + (1,)), - axes=data.getAxisList() + [newdim], - id=data.id) - data = data.reorder('txzy') - with pytest.raises(ValueError): - solver = self.solver_class(data, weights='area') - - # ---------------------------------------------------------------------------- # Constructor tests for the iris interface diff --git a/lib/eofs/tests/test_multivariate_error_handling.py b/lib/eofs/tests/test_multivariate_error_handling.py index 190c848..f48cb18 100644 --- a/lib/eofs/tests/test_multivariate_error_handling.py +++ b/lib/eofs/tests/test_multivariate_error_handling.py @@ -15,13 +15,7 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np -try: - import cdms2 -except ImportError: - pass import pytest import eofs.multivariate as multivariate @@ -32,10 +26,6 @@ # Create a mapping from interface name to solver class. solvers = {'standard': multivariate.standard.MultivariateEof} -try: - solvers['cdms'] = multivariate.cdms.MultivariateEof -except AttributeError: - pass try: solvers['iris'] = multivariate.iris.MultivariateEof except AttributeError: @@ -83,26 +73,6 @@ class TestErrorHandlersStandard(MVErrorHandlersTest): weights = 'equal' -# ---------------------------------------------------------------------------- -# Error Handler tests for the cdms interface - - -class TestErrorHandlersCDMS(MVErrorHandlersTest): - interface = 'cdms' - weights = 'equal' - - def test_projectField_wrong_input_type(self): - solution = reference_multivariate_solution('standard', self.weights) - with pytest.raises(TypeError): - pcs = self.solver.projectField(solution['sst']) - - def test_projectField_time_dimension_not_first(self): - sst1, sst2 = self.solution['sst'] - sst1 = sst1.reorder('-t') - with pytest.raises(ValueError): - pcs = self.solver.projectField([sst1, sst2]) - - # ---------------------------------------------------------------------------- # Error Handler tests for the iris interface @@ -157,47 +127,6 @@ def test_incompatible_weights(self): weights=[weights1, weights2]) -# ---------------------------------------------------------------------------- -# Constructor tests for the cdms interface - - -class TestConstructorCDMS(EofsTest): - """Test the error handling in the cdms interface constructor.""" - - @classmethod - def setup_class(cls): - try: - cls.solver_class = solvers['cdms'] - except KeyError: - pytest.skip('missing dependencies required to test ' - 'the cdms interface') - - def test_wrong_number_weights(self): - solution = reference_multivariate_solution('cdms', 'area') - weights1, weights2 = solution['weights'] - with pytest.raises(ValueError): - solver = self.solver_class(solution['sst'], weights=[weights1]) - - def test_wrong_input_type(self): - solution = reference_multivariate_solution('standard', 'equal') - with pytest.raises(TypeError): - solver = self.solver_class(solution['sst']) - - def test_input_time_dimension_not_first(self): - solution = reference_multivariate_solution('cdms', 'equal') - sst1, sst2 = solution['sst'] - sst1 = sst1.reorder('-t') - with pytest.raises(ValueError): - solver = self.solver_class([sst1, sst2]) - - def test_input_no_spatial_dimensions(self): - solution = reference_multivariate_solution('cdms', 'equal') - sst1, sst2 = solution['sst'] - sst1 = sst1[:, 0, 0] - with pytest.raises(ValueError): - solver = self.solver_class([sst1, sst2]) - - # ---------------------------------------------------------------------------- # Constructor tests for the standard interface diff --git a/lib/eofs/tests/test_multivariate_solution.py b/lib/eofs/tests/test_multivariate_solution.py index 844ff87..d7140e6 100644 --- a/lib/eofs/tests/test_multivariate_solution.py +++ b/lib/eofs/tests/test_multivariate_solution.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np try: from iris.cube import Cube @@ -33,10 +31,6 @@ # Create a mapping from interface name to solver class. solvers = {'standard': multivariate.standard.MultivariateEof} -try: - solvers['cdms'] = multivariate.cdms.MultivariateEof -except AttributeError: - pass try: solvers['iris'] = multivariate.iris.MultivariateEof except AttributeError: @@ -246,39 +240,6 @@ class TestStandardMixedWeights(StandardMVSolutionTest): weights = 'none_area' -# ---------------------------------------------------------------------------- -# Tests for the cdms interface - - -class CDMSMVSolutionTest(MVSolutionTest): - interface = 'cdms' - - def _tomasked(self, value): - try: - return value.asma() - except AttributeError: - return value - - -class TestCDMSEqualWeights(CDMSMVSolutionTest): - weights = 'equal' - - -class TestCDMSLatitudeWeights(CDMSMVSolutionTest): - weights = 'latitude' - alternate_weights_arg = 'coslat' - - -class TestCDMSAreaWeights(CDMSMVSolutionTest): - weights = 'area' - alternate_weights_arg = 'area' - - -class TestCDMSMixedWeights(CDMSMVSolutionTest): - weights = 'none_area' - alternate_weights_arg = (None, 'area') - - # ---------------------------------------------------------------------------- # Tests for the iris interface diff --git a/lib/eofs/tests/test_solution.py b/lib/eofs/tests/test_solution.py index 9dd6586..6eb5c8a 100644 --- a/lib/eofs/tests/test_solution.py +++ b/lib/eofs/tests/test_solution.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np import numpy.ma as ma try: @@ -34,10 +32,6 @@ # Create a mapping from interface name to solver class. solvers = {'standard': eofs.standard.Eof} -try: - solvers['cdms'] = eofs.cdms.Eof -except AttributeError: - pass try: solvers['iris'] = eofs.iris.Eof except AttributeError: @@ -346,84 +340,6 @@ def test_solver_data_is_dask(self): assert isinstance(self.solver._solver._data, dask.array.Array) -# ---------------------------------------------------------------------------- -# Tests for the cdms interface - - -class CDMSSolutionTest(SolutionTest): - """Base class for all cdms interface solution test classes.""" - interface = 'cdms' - - def _tomasked(self, value): - try: - return value.asma() - except AttributeError: - return value - - -class TestCDMSEqualWeights(CDMSSolutionTest): - """Equal grid weighting.""" - weights = 'equal' - - -class TestCDMSLatitudeWeights(CDMSSolutionTest): - """ - Square-root of cosine of latitude grid weighting (automatically - generated weights). - - """ - weights = 'latitude' - alternate_weights_arg = 'coslat' - - -class TestCDMSAreaWeights(CDMSSolutionTest): - """ - Square-root of normalised grid cell area grid weighting - (automatically generated weights). - - """ - weights = 'area' - alternate_weights_arg = 'area' - - -class TestCDMSAreaWeightsTransposedGrid(CDMSSolutionTest): - """ - Square-root of normalised grid cell area grid weighting - (automatically generated weights) after transposing grid variables - to a longitude-latitude grid. - - """ - weights = 'area' - alternate_weights_arg = 'area' - - @classmethod - def modify_solution(cls): - cls.solution['sst'] = cls.solution['sst'].reorder('txy') - cls.solution['eofs'] = cls.solution['eofs'].reorder('-xy') - cls.solution['eofscor'] = cls.solution['eofscor'].reorder('-xy') - cls.solution['eofscov'] = cls.solution['eofscov'].reorder('-xy') - cls.solution['weights'] = cls.solution['weights'].transpose() - cls.solution['rcon'] = cls.solution['rcon'].reorder('-xy') - - -class TestCDMSLatitudeWeightsManual(CDMSSolutionTest): - """ - Square-root of cosine of latitude grid weighting (weights from - reference solution). - - """ - weights = 'latitude' - - -class TestCDMSAreaWeightsManual(CDMSSolutionTest): - """ - Square-root of normalised grid cell area grid weighting (weights - from reference solution). - - """ - weights = 'area' - - # ---------------------------------------------------------------------------- # Tests for the iris interface diff --git a/lib/eofs/tests/test_tools.py b/lib/eofs/tests/test_tools.py index 0e152cf..5bb7874 100644 --- a/lib/eofs/tests/test_tools.py +++ b/lib/eofs/tests/test_tools.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np import numpy.ma as ma try: @@ -35,11 +33,6 @@ # Create a mapping from interface name to tools module and solver class. tools = {'standard': eofs.tools.standard} solvers = {'standard': eofs.standard.Eof} -try: - tools['cdms'] = eofs.tools.cdms - solvers['cdms'] = eofs.cdms.Eof -except AttributeError: - pass try: tools['iris'] = eofs.tools.iris solvers['iris'] = eofs.iris.Eof @@ -140,22 +133,6 @@ def _tomasked(self, value): return value -# ---------------------------------------------------------------------------- -# Tests for the cdms interface - - -class TestToolsCDMS(ToolsTest): - """Test the cdms interface tools.""" - interface = 'cdms' - weights = 'equal' - - def _tomasked(self, value): - try: - return value.asma() - except AttributeError: - return value - - # ---------------------------------------------------------------------------- # Tests for the iris interface diff --git a/lib/eofs/tests/utils.py b/lib/eofs/tests/utils.py index ea2bbaa..71dd146 100644 --- a/lib/eofs/tests/utils.py +++ b/lib/eofs/tests/utils.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np try: from iris.cube import Cube @@ -30,7 +28,7 @@ def _close(a, b, rtol=1e-05, atol=1e-08): def __tomasked(*args): - """Convert cdms2 variables or iris cubes to masked arrays. + """Convert iris cubes to masked arrays. The conversion is safe, so if non-variables/cubes are passed they are just returned. @@ -43,13 +41,6 @@ def __asma(a): a = a.data except NameError: pass - try: - # Retrieve data from cdms variable. - a = a.asma() - except AttributeError: - # The input is already an array or masked array, either extracted - # from an iris cube, or was like that to begin with. - pass return a return [__asma(a) for a in args] diff --git a/lib/eofs/tools/__init__.py b/lib/eofs/tools/__init__.py index d505606..ae1657d 100644 --- a/lib/eofs/tools/__init__.py +++ b/lib/eofs/tools/__init__.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - from . import standard @@ -25,13 +23,13 @@ try: - from . import cdms - __all__.append('cdms') + from . import iris + __all__.append('iris') except ImportError: pass try: - from . import iris - __all__.append('iris') + from . import xarray + __all__.append('xarray') except ImportError: pass diff --git a/lib/eofs/tools/cdms.py b/lib/eofs/tools/cdms.py deleted file mode 100644 index c66ae57..0000000 --- a/lib/eofs/tools/cdms.py +++ /dev/null @@ -1,244 +0,0 @@ -"""Supplementary tools for the `cdms` EOF analysis interface.""" -# (c) Copyright 2010-2013 Andrew Dawson. All Rights Reserved. -# -# This file is part of eofs. -# -# eofs is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# eofs is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# for more details. -# -# You should have received a copy of the GNU General Public License -# along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - -import cdms2 -import numpy as np - -from .standard import covariance_map as standard_covmap -from .standard import correlation_map as standard_cormap -from .generic import covcor_dimensions - - -def _rootcoslat_weights(latdim): - """Square-root of cosine of latitude weights. - - *latdim* - Latitude dimension values. - - """ - coslat = np.cos(np.deg2rad(latdim)) - coslat[np.where(coslat < 0)] = 0. - latw = np.sqrt(coslat) - latw[np.where(np.isnan(latw))] = 0. - return latw - - -def _area_weights(grid, gridorder): - """Area weights. - - *grid* - `cdms2` grid. - - *gridorder* - Either "xy" or "yx". - - """ - latw, lonw = grid.getWeights() - if gridorder == "xy": - wtarray = np.outer(lonw, latw) - else: - wtarray = np.outer(latw, lonw) - wtarray /= wtarray.sum() - wtarray = np.sqrt(wtarray) - return wtarray - - -def weights_array(dataset, scheme): - """Weights for a data set on a grid. - - Returned weights are a `numpy.ndarray` broadcastable to the shape of - the input data. - - **Arguments:** - - *dataset* - A `cdms2` variable to generate weights for. - - *scheme* - Weighting scheme to use. The following values are accepted: - - * *'coslat'* : Square-root of cosine of latitude. - * *'area'* : Square-root of grid cell area normalized by total - grid area. - - **Returns:** - - *weights* - An array contanining the weights (not a `cdms2` variable). - - **Examples:** - - Area weights for a `cdms2` variable on 2D grid: - - weights = weights_array(var2d, scheme='area') - - Square-root of cosine of latitude weights for a `cdms2` variable - with a latitude dimension: - - weights = weights_array(var, scheme='coslat') - - """ - # A re-usable generic error message for the function. When raising an - # exception just fill in what is required. - errstr = "weighting scheme '{!s}' requires {{!s}}".format(scheme) - # Always use lower-case for the scheme, allowing the user to use - # upper-case in their calling code without an error. - scheme = scheme.lower() - if scheme in ('area'): - # Handle area weighting. - grid = dataset.getGrid() - if grid is None: - raise ValueError(errstr.format('a grid')) - order = dataset.getOrder() - if 'xy' in order: - gridorder = 'xy' - dimtoindex = dataset.getLatitude() - elif 'yx' in order: - gridorder = 'yx' - dimtoindex = dataset.getLongitude() - else: - raise ValueError( - errstr.format('adjacent latitude and longitude dimensions')) - # Retrieve area weights for the specified grid. - weights = _area_weights(grid, gridorder) - elif scheme in ('coslat', 'cos_lat'): - # Handle square-root of cosine of latitude weighting. - try: - latdim = dataset.getLatitude()[:] - dimtoindex = dataset.getLatitude() - except (AttributeError, TypeError): - raise ValueError(errstr.format('a latitude dimension')) - # Retrieve latitude weights. - weights = _rootcoslat_weights(latdim) - else: - raise ValueError("invalid weighting scheme: '{!s}'".format(scheme)) - # Re-shape the retrieved weights so that they are broadcastable to the - # shape of the input arrays. This just involves adding any additional - # singleton dimensions to the right of the last weights dimension. - rightdims = len(dataset.shape) - dataset.getAxisIndex(dimtoindex) - 1 - weights = weights.reshape(weights.shape + (1,) * rightdims) - return weights - - -def cdms2_name(variable): - """Return a human-readable name of a cdms2 variable. - - First it tries 'standard_name', then 'long_name' then 'id'. - - """ - names = [getattr(variable, 'standard_name', None), - getattr(variable, 'long_name', None), - getattr(variable, 'id', None)] - try: - return [name for name in names if name is not None][0] - except IndexError: - return 'dataset' - - -def correlation_map(pcs, field): - """Correlation maps for a set of PCs and a spatial-temporal field. - - Given a set of PCs in a `cdms2` variable (e.g., as output from - `eofs.cdms.Eof.pcs`) and a spatial-temporal field in a `cdms` - variable, one correlation map per PC is computed. - - The field must have the same temporal dimension as the PCs. Any - number of spatial dimensions (including zero) are allowed in the - field and there can be any number of PCs. - - **Arguments:** - - *pcs* - PCs in a `cdms2` variable. - - *field* - Spatial-temporal field in a `cdms2` variable. - - **Returns:** - - *correlation_maps* - A `cdms2` variable containing the correlation maps. - - **Examples:** - - Compute correlation maps for each PC:: - - pcs = eofobj.pcs(pcscaling=1) - cormaps = correlation_map(pcs, field) - - """ - cor = standard_cormap(pcs.asma(), field.asma()) - outdims = covcor_dimensions(pcs.getAxisList(), field.getAxisList()) - if not outdims: - # There are no output dimensions, return a scalar. - return cor - # Otherwise return a cdms2 variable with the appropriate dimensions. - cor = cdms2.createVariable(cor, axes=outdims, id='pccor') - cor.long_name = "pc_correlation" - return cor - - -def covariance_map(pcs, field, ddof=1): - """Covariance maps for a set of PCs and a spatial-temporal field. - - Given a set of PCs in a `cdms2` variable (e.g., as output from - `~eofs.cdms.Eof.pcs`) and a spatial-temporal field in a `cdms2` - variable, one covariance map per PC is computed. - - The field must have the same temporal dimension as the PCs. Any - number of spatial dimensions (including zero) are allowed in the - field and there can be any number of PCs. - - **Arguments:** - - *pcs* - PCs in a `cdms2` variable. - - *field* - Spatial-temporal field in a `cdms2` variable. - - **Optional arguments:** - - *ddof* - 'Delta degrees of freedom'. The divisor used to normalize - the covariance matrix is *N - ddof* where *N* is the - number of samples. Defaults to *1*. - - **Returns:** - - *covariance_maps* - A `cdms2` variable containing the covariance maps. - - **Examples:** - - Compute covariance maps for each PC:: - - pcs = eofobj.pcs(pcscaling=1) - covmaps = covariance_map(pcs, field) - - """ - cov = standard_covmap(pcs.asma(), field.asma(), ddof=ddof) - outdims = covcor_dimensions(pcs.getAxisList(), field.getAxisList()) - if not outdims: - # There are no output dimensions, return a scalar. - return cov - # Otherwise return a cdms2 variable with the appropriate dimensions. - cov = cdms2.createVariable(cov, axes=outdims, id='pccov') - cov.long_name = 'pc_covariance' - return cov diff --git a/lib/eofs/tools/generic.py b/lib/eofs/tools/generic.py index 7600033..78c8a53 100644 --- a/lib/eofs/tools/generic.py +++ b/lib/eofs/tools/generic.py @@ -15,7 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa def covcor_dimensions(pc_dims, field_dims): diff --git a/lib/eofs/tools/iris.py b/lib/eofs/tools/iris.py index 323e297..7f1f0bb 100644 --- a/lib/eofs/tools/iris.py +++ b/lib/eofs/tools/iris.py @@ -15,7 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa from copy import copy from functools import reduce import warnings diff --git a/lib/eofs/tools/standard.py b/lib/eofs/tools/standard.py index b4ed837..5569a9e 100644 --- a/lib/eofs/tools/standard.py +++ b/lib/eofs/tools/standard.py @@ -15,8 +15,6 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np import numpy.ma as ma @@ -46,7 +44,7 @@ def _check_flat_center(pcs, field): else: # Record the shape of the field and the number of spatial elements. originalshape = field.shape[1:] - channels = np.product(originalshape) + channels = np.prod(originalshape) # Record the number of PCs. if len(pcs.shape) == 1: npcs = 1 diff --git a/lib/eofs/tools/xarray.py b/lib/eofs/tools/xarray.py index 45d93e5..d1c56dc 100644 --- a/lib/eofs/tools/xarray.py +++ b/lib/eofs/tools/xarray.py @@ -15,13 +15,8 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import numpy as np -try: - import xarray as xr -except ImportError: - import xray as xr +import xarray as xr from . import standard from .generic import covcor_dimensions diff --git a/lib/eofs/xarray.py b/lib/eofs/xarray.py index 70b1c86..c075784 100644 --- a/lib/eofs/xarray.py +++ b/lib/eofs/xarray.py @@ -15,21 +15,16 @@ # # You should have received a copy of the GNU General Public License # along with eofs. If not, see . -from __future__ import (absolute_import, division, print_function) # noqa - import collections.abc -try: - import xarray as xr -except ImportError: - import xray as xr +import xarray as xr from . import standard from .tools.xarray import (find_time_coordinates, categorise_ndcoords, weights_array) -class Eof(object): +class Eof: """EOF analysis (meta-data enabled `xarray` interface)""" def __init__(self, array, weights=None, center=True, ddof=1): diff --git a/pyproject.toml b/pyproject.toml index 8d99309..1ec1801 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dynamic = [ "version", @@ -29,12 +30,8 @@ dependencies = [ "numpy", ] [project.optional-dependencies] -extras = [ - "cdms2", - "cdutil", - "iris", - "xarray", -] +iris = ["scitools-iris"] +xarray = ["xarray"] [project.urls] documentation = "https://ajdawson.github.io/eofs" homepage = "https://github.com/ajdawson/eofs"